history.py
788 lines
| 27.8 KiB
| text/x-python
|
PythonLexer
vivainio
|
r851 | """ History related magics and functionality """ | ||
Fernando Perez
|
r3079 | #----------------------------------------------------------------------------- | ||
Matthias BUSSONNIER
|
r5390 | # Copyright (C) 2010-2011 The IPython Development Team. | ||
Fernando Perez
|
r3079 | # | ||
# Distributed under the terms of the BSD License. | ||||
# | ||||
# The full license is in the file COPYING.txt, distributed with this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
#----------------------------------------------------------------------------- | ||||
from __future__ import print_function | ||||
vivainio
|
r851 | |||
fperez
|
r960 | # Stdlib imports | ||
Thomas Kluyver
|
r3711 | import atexit | ||
Thomas Kluyver
|
r3405 | import datetime | ||
fperez
|
r960 | import os | ||
Thomas Kluyver
|
r3394 | import re | ||
MinRK
|
r5147 | try: | ||
import sqlite3 | ||||
except ImportError: | ||||
sqlite3 = None | ||||
Thomas Kluyver
|
r3711 | import threading | ||
fperez
|
r960 | |||
Fernando Perez
|
r3079 | # Our own packages | ||
Thomas Kluyver
|
r3393 | from IPython.config.configurable import Configurable | ||
MinRK
|
r5147 | from IPython.external.decorator import decorator | ||
Thomas Kluyver
|
r4988 | from IPython.utils.path import locate_profile | ||
MinRK
|
r8165 | from IPython.utils.traitlets import ( | ||
Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError, | ||||
) | ||||
Brian Granger
|
r2498 | from IPython.utils.warn import warn | ||
Fernando Perez
|
r3079 | |||
#----------------------------------------------------------------------------- | ||||
# Classes and functions | ||||
#----------------------------------------------------------------------------- | ||||
MinRK
|
r5147 | class DummyDB(object): | ||
"""Dummy DB that will act as a black hole for history. | ||||
Only used in the absence of sqlite""" | ||||
def execute(*args, **kwargs): | ||||
return [] | ||||
def commit(self, *args, **kwargs): | ||||
pass | ||||
def __enter__(self, *args, **kwargs): | ||||
pass | ||||
def __exit__(self, *args, **kwargs): | ||||
pass | ||||
Fernando Perez
|
r6911 | |||
MinRK
|
r5147 | @decorator | ||
MinRK
|
r8165 | def needs_sqlite(f, self, *a, **kw): | ||
MinRK
|
r5147 | """return an empty list in the absence of sqlite""" | ||
MinRK
|
r8176 | if sqlite3 is None or not self.enabled: | ||
MinRK
|
r5147 | return [] | ||
else: | ||||
MinRK
|
r8165 | return f(self, *a, **kw) | ||
MinRK
|
r5147 | |||
Fernando Perez
|
r6911 | |||
Thomas Kluyver
|
r8461 | if sqlite3 is not None: | ||
DatabaseError = sqlite3.DatabaseError | ||||
else: | ||||
class DatabaseError(Exception): | ||||
"Dummy exception when sqlite could not be imported. Should never occur." | ||||
@decorator | ||||
def catch_corrupt_db(f, self, *a, **kw): | ||||
"""A decorator which wraps HistoryAccessor method calls to catch errors from | ||||
Thomas Kluyver
|
r8462 | a corrupt SQLite database, move the old database out of the way, and create | ||
a new one. | ||||
Thomas Kluyver
|
r8461 | """ | ||
try: | ||||
return f(self, *a, **kw) | ||||
except DatabaseError: | ||||
if os.path.isfile(self.hist_file): | ||||
# Try to move the file out of the way | ||||
base,ext = os.path.splitext(self.hist_file) | ||||
newpath = base + '-corrupt' + ext | ||||
os.rename(self.hist_file, newpath) | ||||
self.init_db() | ||||
print("ERROR! History file wasn't a valid SQLite database.", | ||||
"It was moved to %s" % newpath, "and a new file created.") | ||||
return [] | ||||
else: | ||||
# The hist_file is probably :memory: or something else. | ||||
raise | ||||
Thomas Kluyver
|
r4875 | class HistoryAccessor(Configurable): | ||
Thomas Kluyver
|
r4878 | """Access the history database without adding to it. | ||
This is intended for use by standalone history tools. IPython shells use | ||||
HistoryManager, below, which is a subclass of this.""" | ||||
MinRK
|
r5233 | |||
Thomas Kluyver
|
r3393 | # String holding the path to the history file | ||
MinRK
|
r5233 | hist_file = Unicode(config=True, | ||
help="""Path to file to use for SQLite history database. | ||||
Fernando Perez
|
r6932 | By default, IPython will put the history database in the IPython | ||
profile directory. If you would rather share one history among | ||||
Fernando Perez
|
r6988 | profiles, you can set this value in each, so that they are consistent. | ||
MinRK
|
r5233 | |||
Fernando Perez
|
r6932 | Due to an issue with fcntl, SQLite is known to misbehave on some NFS | ||
mounts. If you see IPython hanging, try setting this to something on a | ||||
local disk, e.g:: | ||||
MinRK
|
r5233 | |||
ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite | ||||
""") | ||||
MinRK
|
r8165 | |||
MinRK
|
r8176 | enabled = Bool(True, config=True, | ||
help="""enable the SQLite history | ||||
MinRK
|
r8165 | |||
MinRK
|
r8176 | set enabled=False to disable the SQLite history, | ||
in which case there will be no stored history, no SQLite connection, | ||||
MinRK
|
r8165 | and no background saving thread. This may be necessary in some | ||
threaded environments where IPython is embedded. | ||||
""" | ||||
) | ||||
connection_options = Dict(config=True, | ||||
help="""Options for configuring the SQLite connection | ||||
These options are passed as keyword args to sqlite3.connect | ||||
when establishing database conenctions. | ||||
""" | ||||
) | ||||
MinRK
|
r5233 | |||
Thomas Kluyver
|
r3388 | # The SQLite database | ||
MinRK
|
r8165 | db = Any() | ||
def _db_changed(self, name, old, new): | ||||
"""validate the db, since it can be an Instance of two different types""" | ||||
Benjie Chen
|
r8217 | connection_types = (DummyDB,) | ||
if sqlite3 is not None: | ||||
connection_types = (DummyDB, sqlite3.Connection) | ||||
if not isinstance(new, connection_types): | ||||
MinRK
|
r8165 | msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \ | ||
(self.__class__.__name__, new) | ||||
raise TraitError(msg) | ||||
Thomas Kluyver
|
r4875 | |||
MinRK
|
r5233 | def __init__(self, profile='default', hist_file=u'', config=None, **traits): | ||
Thomas Kluyver
|
r4875 | """Create a new history accessor. | ||
Thomas Kluyver
|
r4988 | Parameters | ||
---------- | ||||
profile : str | ||||
The name of the profile from which to open history. | ||||
hist_file : str | ||||
Path to an SQLite history database stored by IPython. If specified, | ||||
hist_file overrides profile. | ||||
config : | ||||
Config object. hist_file can also be set through this. | ||||
Fernando Perez
|
r3079 | """ | ||
Fernando Perez
|
r3087 | # We need a pointer back to the shell for various tasks. | ||
MinRK
|
r5233 | super(HistoryAccessor, self).__init__(config=config, **traits) | ||
# defer setting hist_file from kwarg until after init, | ||||
# otherwise the default kwarg value would clobber any value | ||||
# set by config | ||||
if hist_file: | ||||
self.hist_file = hist_file | ||||
Robert Kern
|
r3465 | if self.hist_file == u'': | ||
# No one has set the hist_file, yet. | ||||
Thomas Kluyver
|
r4988 | self.hist_file = self._get_hist_file_name(profile) | ||
Fernando Perez
|
r3079 | |||
MinRK
|
r8176 | if sqlite3 is None and self.enabled: | ||
MinRK
|
r5147 | warn("IPython History requires SQLite, your history will not be saved\n") | ||
MinRK
|
r8176 | self.enabled = False | ||
MinRK
|
r8165 | |||
Thomas Kluyver
|
r8461 | self.init_db() | ||
Thomas Kluyver
|
r4875 | |||
Thomas Kluyver
|
r4988 | def _get_hist_file_name(self, profile='default'): | ||
"""Find the history file for the given profile name. | ||||
This is overridden by the HistoryManager subclass, to use the shell's | ||||
active profile. | ||||
Parameters | ||||
---------- | ||||
profile : str | ||||
The name of a profile which has a history file. | ||||
""" | ||||
return os.path.join(locate_profile(profile), 'history.sqlite') | ||||
Thomas Kluyver
|
r4875 | |||
Thomas Kluyver
|
r8461 | @catch_corrupt_db | ||
Thomas Kluyver
|
r3388 | def init_db(self): | ||
Thomas Kluyver
|
r3405 | """Connect to the database, and create tables if necessary.""" | ||
MinRK
|
r8176 | if not self.enabled: | ||
MinRK
|
r8165 | self.db = DummyDB() | ||
return | ||||
MinRK
|
r4487 | # use detect_types so that timestamps return datetime objects | ||
MinRK
|
r8165 | kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) | ||
kwargs.update(self.connection_options) | ||||
self.db = sqlite3.connect(self.hist_file, **kwargs) | ||||
Thomas Kluyver
|
r3405 | self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer | ||
primary key autoincrement, start timestamp, | ||||
end timestamp, num_cmds integer, remark text)""") | ||||
Bernardo B. Marques
|
r4872 | self.db.execute("""CREATE TABLE IF NOT EXISTS history | ||
Thomas Kluyver
|
r3391 | (session integer, line integer, source text, source_raw text, | ||
PRIMARY KEY (session, line))""") | ||||
# Output history is optional, but ensure the table's there so it can be | ||||
# enabled later. | ||||
self.db.execute("""CREATE TABLE IF NOT EXISTS output_history | ||||
(session integer, line integer, output text, | ||||
Thomas Kluyver
|
r3388 | PRIMARY KEY (session, line))""") | ||
self.db.commit() | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r4876 | def writeout_cache(self): | ||
"""Overridden by HistoryManager to dump the cache before certain | ||||
database lookups.""" | ||||
pass | ||||
Thomas Kluyver
|
r3405 | ## ------------------------------- | ||
## Methods for retrieving history: | ||||
## ------------------------------- | ||||
Thomas Kluyver
|
r3435 | def _run_sql(self, sql, params, raw=True, output=False): | ||
Thomas Kluyver
|
r3401 | """Prepares and runs an SQL query for the history database. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3401 | Parameters | ||
---------- | ||||
sql : str | ||||
Any filtering expressions to go after SELECT ... FROM ... | ||||
params : tuple | ||||
Parameters passed to the SQL query (to replace "?") | ||||
Thomas Kluyver
|
r3435 | raw, output : bool | ||
See :meth:`get_range` | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3401 | Returns | ||
------- | ||||
Thomas Kluyver
|
r3435 | Tuples as :meth:`get_range` | ||
Thomas Kluyver
|
r3401 | """ | ||
Thomas Kluyver
|
r3388 | toget = 'source_raw' if raw else 'source' | ||
Thomas Kluyver
|
r3397 | sqlfrom = "history" | ||
if output: | ||||
sqlfrom = "history LEFT JOIN output_history USING (session, line)" | ||||
toget = "history.%s, output_history.output" % toget | ||||
Thomas Kluyver
|
r3401 | cur = self.db.execute("SELECT session, line, %s FROM %s " %\ | ||
(toget, sqlfrom) + sql, params) | ||||
Thomas Kluyver
|
r3415 | if output: # Regroup into 3-tuples, and parse JSON | ||
Thomas Kluyver
|
r3741 | return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) | ||
Thomas Kluyver
|
r3401 | return cur | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r5147 | @needs_sqlite | ||
Thomas Kluyver
|
r8461 | @catch_corrupt_db | ||
MinRK
|
r4487 | def get_session_info(self, session=0): | ||
"""get info about a session | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | Parameters | ||
---------- | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | session : int | ||
Session number to retrieve. The current session is 0, and negative | ||||
numbers count back from current session, so -1 is previous session. | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | Returns | ||
------- | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r6932 | (session_id [int], start [datetime], end [datetime], num_cmds [int], | ||
remark [unicode]) | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | Sessions that are running or did not exit cleanly will have `end=None` | ||
and `num_cmds=None`. | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | """ | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | if session <= 0: | ||
session += self.session_number | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4487 | query = "SELECT * from sessions where session == ?" | ||
return self.db.execute(query, (session,)).fetchone() | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r8463 | @catch_corrupt_db | ||
Thomas Kluyver
|
r3435 | def get_tail(self, n=10, raw=True, output=False, include_latest=False): | ||
Thomas Kluyver
|
r3420 | """Get the last n lines from the history database. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Parameters | ||
---------- | ||||
n : int | ||||
The number of lines to get | ||||
raw, output : bool | ||||
See :meth:`get_range` | ||||
include_latest : bool | ||||
If False (default), n+1 lines are fetched, and the latest one | ||||
is discarded. This is intended to be used where the function | ||||
is called by a user command, which it should not return. | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Returns | ||
------- | ||||
Tuples as :meth:`get_range` | ||||
""" | ||||
Thomas Kluyver
|
r3403 | self.writeout_cache() | ||
Thomas Kluyver
|
r3420 | if not include_latest: | ||
n += 1 | ||||
Thomas Kluyver
|
r3435 | cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?", | ||
Thomas Kluyver
|
r3401 | (n,), raw=raw, output=output) | ||
Thomas Kluyver
|
r3420 | if not include_latest: | ||
return reversed(list(cur)[1:]) | ||||
Thomas Kluyver
|
r3401 | return reversed(list(cur)) | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r8463 | @catch_corrupt_db | ||
Thomas Kluyver
|
r3435 | def search(self, pattern="*", raw=True, search_raw=True, | ||
Takafumi Arakaki
|
r8399 | output=False, n=None): | ||
Thomas Kluyver
|
r3420 | """Search the database using unix glob-style matching (wildcards | ||
* and ?). | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Parameters | ||
---------- | ||||
pattern : str | ||||
The wildcarded pattern to match when searching | ||||
search_raw : bool | ||||
If True, search the raw input, otherwise, the parsed input | ||||
raw, output : bool | ||||
See :meth:`get_range` | ||||
Takafumi Arakaki
|
r8399 | n : None or int | ||
If an integer is given, it defines the limit of | ||||
returned entries. | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3388 | Returns | ||
------- | ||||
Thomas Kluyver
|
r3435 | Tuples as :meth:`get_range` | ||
Thomas Kluyver
|
r3388 | """ | ||
Thomas Kluyver
|
r3418 | tosearch = "source_raw" if search_raw else "source" | ||
Thomas Kluyver
|
r3400 | if output: | ||
tosearch = "history." + tosearch | ||||
Thomas Kluyver
|
r3403 | self.writeout_cache() | ||
Takafumi Arakaki
|
r8399 | sqlform = "WHERE %s GLOB ?" % tosearch | ||
params = (pattern,) | ||||
if n is not None: | ||||
sqlform += " ORDER BY session DESC, line DESC LIMIT ?" | ||||
params += (n,) | ||||
cur = self._run_sql(sqlform, params, raw=raw, output=output) | ||||
if n is not None: | ||||
Takafumi Arakaki
|
r8403 | return reversed(list(cur)) | ||
Takafumi Arakaki
|
r8399 | return cur | ||
Thomas Kluyver
|
r8463 | |||
@catch_corrupt_db | ||||
Thomas Kluyver
|
r4876 | def get_range(self, session, start=1, stop=None, raw=True,output=False): | ||
Thomas Kluyver
|
r3394 | """Retrieve input by session. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3394 | Parameters | ||
---------- | ||||
session : int | ||||
Thomas Kluyver
|
r4876 | Session number to retrieve. | ||
Thomas Kluyver
|
r3394 | start : int | ||
First line to retrieve. | ||||
stop : int | ||||
Thomas Kluyver
|
r3396 | End of line range (excluded from output itself). If None, retrieve | ||
to the end of the session. | ||||
Thomas Kluyver
|
r3394 | raw : bool | ||
If True, return untranslated input | ||||
output : bool | ||||
If True, attempt to include output. This will be 'real' Python | ||||
objects for the current session, or text reprs from previous | ||||
sessions if db_log_output was enabled at the time. Where no output | ||||
is found, None is used. | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3394 | Returns | ||
------- | ||||
An iterator over the desired lines. Each line is a 3-tuple, either | ||||
(session, line, input) if output is False, or | ||||
(session, line, (input, output)) if output is True. | ||||
""" | ||||
if stop: | ||||
Thomas Kluyver
|
r3395 | lineclause = "line >= ? AND line < ?" | ||
Thomas Kluyver
|
r3394 | params = (session, start, stop) | ||
else: | ||||
lineclause = "line>=?" | ||||
params = (session, start) | ||||
Bernardo B. Marques
|
r4872 | |||
Takafumi Arakaki
|
r8397 | return self._run_sql("WHERE session==? AND %s" % lineclause, | ||
Thomas Kluyver
|
r3401 | params, raw=raw, output=output) | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | def get_range_by_str(self, rangestr, raw=True, output=False): | ||
Thomas Kluyver
|
r3394 | """Get lines of history from a string of ranges, as used by magic | ||
Thomas Kluyver
|
r3435 | commands %hist, %save, %macro, etc. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Parameters | ||
---------- | ||||
rangestr : str | ||||
A string specifying ranges, e.g. "5 ~2/1-4". See | ||||
:func:`magic_history` for full details. | ||||
raw, output : bool | ||||
As :meth:`get_range` | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Returns | ||
------- | ||||
Tuples as :meth:`get_range` | ||||
""" | ||||
Thomas Kluyver
|
r3395 | for sess, s, e in extract_hist_ranges(rangestr): | ||
Thomas Kluyver
|
r3435 | for line in self.get_range(sess, s, e, raw=raw, output=output): | ||
Thomas Kluyver
|
r3394 | yield line | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r4875 | |||
class HistoryManager(HistoryAccessor): | ||||
"""A class to organize all history-related functionality in one place. | ||||
""" | ||||
# Public interface | ||||
# An instance of the IPython shell we are attached to | ||||
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') | ||||
# Lists to hold processed and raw history. These start with a blank entry | ||||
# so that we can index them starting from 1 | ||||
input_hist_parsed = List([""]) | ||||
input_hist_raw = List([""]) | ||||
# A list of directories visited during session | ||||
dir_hist = List() | ||||
def _dir_hist_default(self): | ||||
try: | ||||
return [os.getcwdu()] | ||||
except OSError: | ||||
return [] | ||||
# A dict of output history, keyed with ints from the shell's | ||||
# execution count. | ||||
output_hist = Dict() | ||||
# The text/plain repr of outputs. | ||||
output_hist_reprs = Dict() | ||||
# The number of the current session in the history database | ||||
MinRK
|
r5344 | session_number = Integer() | ||
Thomas Kluyver
|
r4875 | # Should we log output to the database? (default no) | ||
db_log_output = Bool(False, config=True) | ||||
# Write to database every x commands (higher values save disk access & power) | ||||
# Values of 1 or less effectively disable caching. | ||||
MinRK
|
r5344 | db_cache_size = Integer(0, config=True) | ||
Thomas Kluyver
|
r4875 | # The input and output caches | ||
db_input_cache = List() | ||||
db_output_cache = List() | ||||
# History saving in separate thread | ||||
save_thread = Instance('IPython.core.history.HistorySavingThread') | ||||
try: # Event is a function returning an instance of _Event... | ||||
save_flag = Instance(threading._Event) | ||||
except AttributeError: # ...until Python 3.3, when it's a class. | ||||
save_flag = Instance(threading.Event) | ||||
# Private interface | ||||
# Variables used to store the three last inputs from the user. On each new | ||||
# history update, we populate the user's namespace with these, shifted as | ||||
# necessary. | ||||
_i00 = Unicode(u'') | ||||
_i = Unicode(u'') | ||||
_ii = Unicode(u'') | ||||
_iii = Unicode(u'') | ||||
# A regex matching all forms of the exit command, so that we don't store | ||||
# them in the history (it's annoying to rewind the first entry and land on | ||||
# an exit call). | ||||
_exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") | ||||
def __init__(self, shell=None, config=None, **traits): | ||||
"""Create a new history manager associated with a shell instance. | ||||
""" | ||||
# We need a pointer back to the shell for various tasks. | ||||
super(HistoryManager, self).__init__(shell=shell, config=config, | ||||
**traits) | ||||
self.save_flag = threading.Event() | ||||
self.db_input_cache_lock = threading.Lock() | ||||
self.db_output_cache_lock = threading.Lock() | ||||
MinRK
|
r8176 | if self.enabled and self.hist_file != ':memory:': | ||
MinRK
|
r6871 | self.save_thread = HistorySavingThread(self) | ||
self.save_thread.start() | ||||
Thomas Kluyver
|
r4875 | |||
self.new_session() | ||||
MinRK
|
r5147 | |||
Thomas Kluyver
|
r4988 | def _get_hist_file_name(self, profile=None): | ||
"""Get default history file name based on the Shell's profile. | ||||
The profile parameter is ignored, but must exist for compatibility with | ||||
the parent class.""" | ||||
Thomas Kluyver
|
r4875 | profile_dir = self.shell.profile_dir.location | ||
return os.path.join(profile_dir, 'history.sqlite') | ||||
MinRK
|
r5147 | @needs_sqlite | ||
Thomas Kluyver
|
r4875 | def new_session(self, conn=None): | ||
"""Get a new session number.""" | ||||
if conn is None: | ||||
conn = self.db | ||||
with conn: | ||||
cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL, | ||||
NULL, "") """, (datetime.datetime.now(),)) | ||||
self.session_number = cur.lastrowid | ||||
def end_session(self): | ||||
"""Close the database session, filling in the end time and line count.""" | ||||
self.writeout_cache() | ||||
with self.db: | ||||
self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE | ||||
session==?""", (datetime.datetime.now(), | ||||
len(self.input_hist_parsed)-1, self.session_number)) | ||||
self.session_number = 0 | ||||
def name_session(self, name): | ||||
"""Give the current session a name in the history database.""" | ||||
with self.db: | ||||
self.db.execute("UPDATE sessions SET remark=? WHERE session==?", | ||||
(name, self.session_number)) | ||||
def reset(self, new_session=True): | ||||
"""Clear the session history, releasing all object references, and | ||||
optionally open a new session.""" | ||||
self.output_hist.clear() | ||||
# The directory history can't be completely empty | ||||
self.dir_hist[:] = [os.getcwdu()] | ||||
if new_session: | ||||
if self.session_number: | ||||
self.end_session() | ||||
self.input_hist_parsed[:] = [""] | ||||
self.input_hist_raw[:] = [""] | ||||
self.new_session() | ||||
# ------------------------------ | ||||
# Methods for retrieving history | ||||
# ------------------------------ | ||||
def _get_range_session(self, start=1, stop=None, raw=True, output=False): | ||||
"""Get input and output history from the current session. Called by | ||||
get_range, and takes similar parameters.""" | ||||
input_hist = self.input_hist_raw if raw else self.input_hist_parsed | ||||
n = len(input_hist) | ||||
if start < 0: | ||||
start += n | ||||
if not stop or (stop > n): | ||||
stop = n | ||||
elif stop < 0: | ||||
stop += n | ||||
for i in range(start, stop): | ||||
if output: | ||||
line = (input_hist[i], self.output_hist_reprs.get(i)) | ||||
else: | ||||
line = input_hist[i] | ||||
yield (0, i, line) | ||||
Thomas Kluyver
|
r4876 | |||
Thomas Kluyver
|
r4877 | def get_range(self, session=0, start=1, stop=None, raw=True,output=False): | ||
Thomas Kluyver
|
r4876 | """Retrieve input by session. | ||
Parameters | ||||
---------- | ||||
session : int | ||||
Session number to retrieve. The current session is 0, and negative | ||||
numbers count back from current session, so -1 is previous session. | ||||
start : int | ||||
First line to retrieve. | ||||
stop : int | ||||
End of line range (excluded from output itself). If None, retrieve | ||||
to the end of the session. | ||||
raw : bool | ||||
If True, return untranslated input | ||||
output : bool | ||||
If True, attempt to include output. This will be 'real' Python | ||||
objects for the current session, or text reprs from previous | ||||
sessions if db_log_output was enabled at the time. Where no output | ||||
is found, None is used. | ||||
Returns | ||||
------- | ||||
An iterator over the desired lines. Each line is a 3-tuple, either | ||||
(session, line, input) if output is False, or | ||||
(session, line, (input, output)) if output is True. | ||||
""" | ||||
if session <= 0: | ||||
session += self.session_number | ||||
if session==self.session_number: # Current session | ||||
return self._get_range_session(start, stop, raw, output) | ||||
Fernando Perez
|
r6932 | return super(HistoryManager, self).get_range(session, start, stop, raw, | ||
output) | ||||
Thomas Kluyver
|
r4875 | |||
Thomas Kluyver
|
r3405 | ## ---------------------------- | ||
## Methods for storing history: | ||||
## ---------------------------- | ||||
Thomas Kluyver
|
r3390 | def store_inputs(self, line_num, source, source_raw=None): | ||
Fernando Perez
|
r3087 | """Store source and raw input in history and create input cache | ||
variables _i*. | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r3080 | Parameters | ||
---------- | ||||
Thomas Kluyver
|
r3390 | line_num : int | ||
The prompt number of this input. | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r3080 | source : str | ||
Python input. | ||||
source_raw : str, optional | ||||
If given, this is the raw input without any IPython transformations | ||||
applied to it. If not given, ``source`` is used. | ||||
""" | ||||
if source_raw is None: | ||||
source_raw = source | ||||
Thomas Kluyver
|
r3419 | source = source.rstrip('\n') | ||
source_raw = source_raw.rstrip('\n') | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r3246 | # do not store exit/quit commands | ||
Thomas Kluyver
|
r3750 | if self._exit_re.match(source_raw.strip()): | ||
Satrajit Ghosh
|
r3243 | return | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3418 | self.input_hist_parsed.append(source) | ||
self.input_hist_raw.append(source_raw) | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3713 | with self.db_input_cache_lock: | ||
self.db_input_cache.append((line_num, source, source_raw)) | ||||
# Trigger to flush cache and write to DB. | ||||
if len(self.db_input_cache) >= self.db_cache_size: | ||||
self.save_flag.set() | ||||
Fernando Perez
|
r3080 | |||
Fernando Perez
|
r3087 | # update the auto _i variables | ||
self._iii = self._ii | ||||
self._ii = self._i | ||||
self._i = self._i00 | ||||
self._i00 = source_raw | ||||
# hackish access to user namespace to create _i1,_i2... dynamically | ||||
Thomas Kluyver
|
r3390 | new_i = '_i%s' % line_num | ||
Fernando Perez
|
r3087 | to_main = {'_i': self._i, | ||
'_ii': self._ii, | ||||
'_iii': self._iii, | ||||
new_i : self._i00 } | ||||
Thomas Kluyver
|
r5458 | |||
self.shell.push(to_main, interactive=False) | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3415 | def store_output(self, line_num): | ||
Thomas Kluyver
|
r3417 | """If database output logging is enabled, this saves all the | ||
outputs from the indicated prompt number to the database. It's | ||||
Thomas Kluyver
|
r3435 | called by run_cell after code has been executed. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3435 | Parameters | ||
---------- | ||||
line_num : int | ||||
The line number from which to save outputs | ||||
""" | ||||
Thomas Kluyver
|
r3741 | if (not self.db_log_output) or (line_num not in self.output_hist_reprs): | ||
Thomas Kluyver
|
r3391 | return | ||
Thomas Kluyver
|
r3741 | output = self.output_hist_reprs[line_num] | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3713 | with self.db_output_cache_lock: | ||
self.db_output_cache.append((line_num, output)) | ||||
Thomas Kluyver
|
r3437 | if self.db_cache_size <= 1: | ||
Thomas Kluyver
|
r3711 | self.save_flag.set() | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3711 | def _writeout_input_cache(self, conn): | ||
with conn: | ||||
Thomas Kluyver
|
r3700 | for line in self.db_input_cache: | ||
Thomas Kluyver
|
r3711 | conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)", | ||
Thomas Kluyver
|
r3437 | (self.session_number,)+line) | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3711 | def _writeout_output_cache(self, conn): | ||
with conn: | ||||
Thomas Kluyver
|
r3700 | for line in self.db_output_cache: | ||
Thomas Kluyver
|
r3711 | conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", | ||
Thomas Kluyver
|
r3437 | (self.session_number,)+line) | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r5147 | @needs_sqlite | ||
Thomas Kluyver
|
r3711 | def writeout_cache(self, conn=None): | ||
Thomas Kluyver
|
r3437 | """Write any entries in the cache to the database.""" | ||
Thomas Kluyver
|
r3711 | if conn is None: | ||
conn = self.db | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3713 | with self.db_input_cache_lock: | ||
try: | ||||
Thomas Kluyver
|
r3711 | self._writeout_input_cache(conn) | ||
Thomas Kluyver
|
r3437 | except sqlite3.IntegrityError: | ||
Thomas Kluyver
|
r3714 | self.new_session(conn) | ||
Thomas Kluyver
|
r3713 | print("ERROR! Session/line number was not unique in", | ||
"database. History logging moved to new session", | ||||
self.session_number) | ||||
Fernando Perez
|
r6932 | try: | ||
# Try writing to the new session. If this fails, don't | ||||
# recurse | ||||
Thomas Kluyver
|
r3713 | self._writeout_input_cache(conn) | ||
except sqlite3.IntegrityError: | ||||
pass | ||||
finally: | ||||
self.db_input_cache = [] | ||||
with self.db_output_cache_lock: | ||||
try: | ||||
self._writeout_output_cache(conn) | ||||
except sqlite3.IntegrityError: | ||||
print("!! Session/line number for output was not unique", | ||||
"in database. Output will not be stored.") | ||||
finally: | ||||
self.db_output_cache = [] | ||||
Fernando Perez
|
r3087 | |||
Thomas Kluyver
|
r3711 | |||
class HistorySavingThread(threading.Thread): | ||||
Thomas Kluyver
|
r3716 | """This thread takes care of writing history to the database, so that | ||
the UI isn't held up while that happens. | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3716 | It waits for the HistoryManager's save_flag to be set, then writes out | ||
the history cache. The main thread is responsible for setting the flag when | ||||
the cache size reaches a defined threshold.""" | ||||
Thomas Kluyver
|
r3711 | daemon = True | ||
stop_now = False | ||||
MinRK
|
r8176 | enabled = True | ||
Thomas Kluyver
|
r3711 | def __init__(self, history_manager): | ||
super(HistorySavingThread, self).__init__() | ||||
self.history_manager = history_manager | ||||
MinRK
|
r8176 | self.enabled = history_manager.enabled | ||
Thomas Kluyver
|
r3711 | atexit.register(self.stop) | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r5147 | @needs_sqlite | ||
Thomas Kluyver
|
r3711 | def run(self): | ||
# We need a separate db connection per thread: | ||||
Thomas Kluyver
|
r3720 | try: | ||
MinRK
|
r8165 | self.db = sqlite3.connect(self.history_manager.hist_file, | ||
**self.history_manager.connection_options | ||||
) | ||||
Thomas Kluyver
|
r3720 | while True: | ||
self.history_manager.save_flag.wait() | ||||
if self.stop_now: | ||||
return | ||||
self.history_manager.save_flag.clear() | ||||
self.history_manager.writeout_cache(self.db) | ||||
except Exception as e: | ||||
print(("The history saving thread hit an unexpected error (%s)." | ||||
"History will not be written to the database.") % repr(e)) | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3711 | def stop(self): | ||
Thomas Kluyver
|
r3716 | """This can be called from the main thread to safely stop this thread. | ||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3716 | Note that it does not attempt to write out remaining history before | ||
exiting. That should be done by calling the HistoryManager's | ||||
end_session method.""" | ||||
Thomas Kluyver
|
r3711 | self.stop_now = True | ||
self.history_manager.save_flag.set() | ||||
Thomas Kluyver
|
r3716 | self.join() | ||
Thomas Kluyver
|
r3711 | |||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3395 | # To match, e.g. ~5/8-~2/3 | ||
Thomas Kluyver
|
r3394 | range_re = re.compile(r""" | ||
Thomas Kluyver
|
r3395 | ((?P<startsess>~?\d+)/)? | ||
Thomas Kluyver
|
r3394 | (?P<start>\d+) # Only the start line num is compulsory | ||
((?P<sep>[\-:]) | ||||
Thomas Kluyver
|
r3395 | ((?P<endsess>~?\d+)/)? | ||
Thomas Kluyver
|
r3394 | (?P<end>\d+))? | ||
Thomas Kluyver
|
r3493 | $""", re.VERBOSE) | ||
Thomas Kluyver
|
r3394 | |||
Fernando Perez
|
r6911 | |||
Thomas Kluyver
|
r3394 | def extract_hist_ranges(ranges_str): | ||
"""Turn a string of history ranges into 3-tuples of (session, start, stop). | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3394 | Examples | ||
-------- | ||||
Thomas Kluyver
|
r3395 | list(extract_input_ranges("~8/5-~7/4 2")) | ||
Thomas Kluyver
|
r3394 | [(-8, 5, None), (-7, 1, 4), (0, 2, 3)] | ||
""" | ||||
for range_str in ranges_str.split(): | ||||
rmatch = range_re.match(range_str) | ||||
Thomas Kluyver
|
r3416 | if not rmatch: | ||
continue | ||||
Thomas Kluyver
|
r3394 | start = int(rmatch.group("start")) | ||
end = rmatch.group("end") | ||||
end = int(end) if end else start+1 # If no end specified, get (a, a+1) | ||||
if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3] | ||||
end += 1 | ||||
startsess = rmatch.group("startsess") or "0" | ||||
endsess = rmatch.group("endsess") or startsess | ||||
startsess = int(startsess.replace("~","-")) | ||||
endsess = int(endsess.replace("~","-")) | ||||
assert endsess >= startsess | ||||
if endsess == startsess: | ||||
yield (startsess, start, end) | ||||
continue | ||||
# Multiple sessions in one range: | ||||
yield (startsess, start, None) | ||||
for sess in range(startsess+1, endsess): | ||||
yield (sess, 1, None) | ||||
yield (endsess, 1, end) | ||||
Fernando Perez
|
r6911 | |||
Thomas Kluyver
|
r3394 | def _format_lineno(session, line): | ||
"""Helper function to format line numbers properly.""" | ||||
if session == 0: | ||||
return str(line) | ||||
return "%s#%s" % (session, line) | ||||
vivainio
|
r851 | |||
Fernando Perez
|
r6911 | |||