history.py
972 lines
| 33.5 KiB
| text/x-python
|
PythonLexer
|
r851 | """ History related magics and functionality """ | ||
|
r3079 | |||
|
r22264 | # Copyright (c) IPython Development Team. | ||
# Distributed under the terms of the Modified BSD License. | ||||
|
r851 | |||
|
r3711 | import atexit | ||
|
r3405 | import datetime | ||
|
r26129 | from pathlib import Path | ||
|
r3394 | import re | ||
|
r25519 | import sqlite3 | ||
|
r3711 | import threading | ||
|
r960 | |||
|
r22182 | from traitlets.config.configurable import LoggingConfigurable | ||
|
r20813 | from decorator import decorator | ||
|
r16033 | from IPython.utils.decorators import undoc | ||
|
r24224 | from IPython.paths import locate_profile | ||
|
r21253 | from traitlets import ( | ||
|
r26131 | Any, | ||
Bool, | ||||
Dict, | ||||
Instance, | ||||
Integer, | ||||
List, | ||||
Unicode, | ||||
Union, | ||||
TraitError, | ||||
default, | ||||
observe, | ||||
|
r8165 | ) | ||
|
r3079 | |||
#----------------------------------------------------------------------------- | ||||
# Classes and functions | ||||
#----------------------------------------------------------------------------- | ||||
|
r16033 | @undoc | ||
|
r5147 | class DummyDB(object): | ||
"""Dummy DB that will act as a black hole for history. | ||||
|
r26129 | |||
|
r5147 | Only used in the absence of sqlite""" | ||
def execute(*args, **kwargs): | ||||
return [] | ||||
|
r26129 | |||
|
r5147 | def commit(self, *args, **kwargs): | ||
pass | ||||
|
r26129 | |||
|
r5147 | def __enter__(self, *args, **kwargs): | ||
pass | ||||
|
r26129 | |||
|
r5147 | def __exit__(self, *args, **kwargs): | ||
pass | ||||
|
r6911 | |||
|
r25522 | @decorator | ||
|
r25523 | def only_when_enabled(f, self, *a, **kw): | ||
|
r25522 | """Decorator: return an empty list in the absence of sqlite.""" | ||
if not self.enabled: | ||||
return [] | ||||
else: | ||||
return f(self, *a, **kw) | ||||
|
r8461 | |||
|
r22271 | # use 16kB as threshold for whether a corrupt history db should be saved | ||
# that should be at least 100 entries or so | ||||
_SAVE_DB_SIZE = 16384 | ||||
|
r8461 | @decorator | ||
def catch_corrupt_db(f, self, *a, **kw): | ||||
"""A decorator which wraps HistoryAccessor method calls to catch errors from | ||||
|
r8462 | a corrupt SQLite database, move the old database out of the way, and create | ||
a new one. | ||||
|
r22271 | |||
We avoid clobbering larger databases because this may be triggered due to filesystem issues, | ||||
not just a corrupt file. | ||||
|
r8461 | """ | ||
try: | ||||
return f(self, *a, **kw) | ||||
|
r25522 | except (sqlite3.DatabaseError, sqlite3.OperationalError) as e: | ||
|
r22270 | self._corrupt_db_counter += 1 | ||
self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e) | ||||
if self.hist_file != ':memory:': | ||||
if self._corrupt_db_counter > self._corrupt_db_limit: | ||||
self.hist_file = ':memory:' | ||||
self.log.error("Failed to load history too many times, history will not be saved.") | ||||
|
r26129 | elif self.hist_file.is_file(): | ||
|
r22271 | # move the file out of the way | ||
|
r26129 | base = str(self.hist_file.parent / self.hist_file.stem) | ||
ext = self.hist_file.suffix | ||||
size = self.hist_file.stat().st_size | ||||
|
r22271 | if size >= _SAVE_DB_SIZE: | ||
# if there's significant content, avoid clobbering | ||||
now = datetime.datetime.now().isoformat().replace(':', '.') | ||||
newpath = base + '-corrupt-' + now + ext | ||||
# don't clobber previous corrupt backups | ||||
for i in range(100): | ||||
|
r26129 | if not Path(newpath).exists(): | ||
|
r22271 | break | ||
else: | ||||
newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext | ||||
else: | ||||
# not much content, possibly empty; don't worry about clobbering | ||||
# maybe we should just delete it? | ||||
newpath = base + '-corrupt' + ext | ||||
|
r26129 | self.hist_file.rename(newpath) | ||
|
r22270 | self.log.error("History file was moved to %s and a new file created.", newpath) | ||
|
r8461 | self.init_db() | ||
return [] | ||||
else: | ||||
|
r22270 | # Failed with :memory:, something serious is wrong | ||
|
r8461 | raise | ||
|
r26129 | |||
|
r26131 | |||
|
r22182 | class HistoryAccessorBase(LoggingConfigurable): | ||
|
r18846 | """An abstract class for History Accessors """ | ||
|
r18815 | |||
def get_tail(self, n=10, raw=True, output=False, include_latest=False): | ||||
|
r18848 | raise NotImplementedError | ||
|
r18815 | |||
def search(self, pattern="*", raw=True, search_raw=True, | ||||
output=False, n=None, unique=False): | ||||
|
r18848 | raise NotImplementedError | ||
|
r18815 | |||
def get_range(self, session, start=1, stop=None, raw=True,output=False): | ||||
|
r18848 | raise NotImplementedError | ||
|
r8461 | |||
|
r18846 | def get_range_by_str(self, rangestr, raw=True, output=False): | ||
|
r18848 | raise NotImplementedError | ||
|
r8461 | |||
|
r18815 | class HistoryAccessor(HistoryAccessorBase): | ||
|
r4878 | """Access the history database without adding to it. | ||
|
r26129 | |||
|
r4878 | This is intended for use by standalone history tools. IPython shells use | ||
HistoryManager, below, which is a subclass of this.""" | ||||
|
r5233 | |||
|
r22270 | # counter for init_db retries, so we don't keep trying over and over | ||
_corrupt_db_counter = 0 | ||||
# after two failures, fallback on :memory: | ||||
_corrupt_db_limit = 2 | ||||
|
r3393 | # String holding the path to the history file | ||
|
r26131 | hist_file = Union( | ||
[Instance(Path), Unicode()], | ||||
|
r5233 | help="""Path to file to use for SQLite history database. | ||
|
r26129 | |||
|
r6932 | By default, IPython will put the history database in the IPython | ||
profile directory. If you would rather share one history among | ||||
|
r6988 | profiles, you can set this value in each, so that they are consistent. | ||
|
r26129 | |||
|
r6932 | Due to an issue with fcntl, SQLite is known to misbehave on some NFS | ||
mounts. If you see IPython hanging, try setting this to something on a | ||||
local disk, e.g:: | ||||
|
r26129 | |||
|
r5233 | ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite | ||
|
r22634 | |||
you can also use the specific value `:memory:` (including the colon | ||||
at both end but not the back ticks), to avoid creating an history file. | ||||
|
r26129 | |||
|
r26131 | """, | ||
|
r26129 | ).tag(config=True) | ||
|
r22340 | enabled = Bool(True, | ||
|
r8176 | help="""enable the SQLite history | ||
|
r26129 | |||
|
r8176 | set enabled=False to disable the SQLite history, | ||
in which case there will be no stored history, no SQLite connection, | ||||
|
r8165 | and no background saving thread. This may be necessary in some | ||
threaded environments where IPython is embedded. | ||||
|
r27639 | """, | ||
|
r22340 | ).tag(config=True) | ||
|
r26129 | |||
|
r22340 | connection_options = Dict( | ||
|
r8165 | help="""Options for configuring the SQLite connection | ||
|
r26129 | |||
|
r8165 | These options are passed as keyword args to sqlite3.connect | ||
|
r24132 | when establishing database connections. | ||
|
r8165 | """ | ||
|
r22340 | ).tag(config=True) | ||
|
r5233 | |||
|
r28013 | @default("connection_options") | ||
def _default_connection_options(self): | ||||
return dict(check_same_thread=False) | ||||
|
r3388 | # The SQLite database | ||
|
r8165 | db = Any() | ||
|
r22340 | @observe('db') | ||
def _db_changed(self, change): | ||||
|
r8165 | """validate the db, since it can be an Instance of two different types""" | ||
|
r22340 | new = change['new'] | ||
|
r25522 | connection_types = (DummyDB, sqlite3.Connection) | ||
|
r8217 | if not isinstance(new, connection_types): | ||
|
r8165 | msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \ | ||
(self.__class__.__name__, new) | ||||
raise TraitError(msg) | ||||
|
r26129 | |||
|
r26131 | def __init__(self, profile="default", hist_file="", **traits): | ||
|
r4875 | """Create a new history accessor. | ||
|
r26129 | |||
|
r4988 | Parameters | ||
---------- | ||||
profile : str | ||||
|
r27290 | The name of the profile from which to open history. | ||
|
r4988 | hist_file : str | ||
|
r27290 | Path to an SQLite history database stored by IPython. If specified, | ||
hist_file overrides profile. | ||||
|
r21253 | config : :class:`~traitlets.config.loader.Config` | ||
|
r27290 | Config object. hist_file can also be set through this. | ||
|
r3079 | """ | ||
|
r11064 | super(HistoryAccessor, self).__init__(**traits) | ||
|
r5233 | # defer setting hist_file from kwarg until after init, | ||
# otherwise the default kwarg value would clobber any value | ||||
# set by config | ||||
if hist_file: | ||||
self.hist_file = hist_file | ||||
|
r26129 | |||
try: | ||||
|
r26131 | self.hist_file | ||
|
r26129 | except TraitError: | ||
|
r3465 | # No one has set the hist_file, yet. | ||
|
r4988 | self.hist_file = self._get_hist_file_name(profile) | ||
|
r26129 | |||
|
r8461 | self.init_db() | ||
|
r26129 | |||
|
r4988 | def _get_hist_file_name(self, profile='default'): | ||
"""Find the history file for the given profile name. | ||||
|
r26129 | |||
|
r4988 | This is overridden by the HistoryManager subclass, to use the shell's | ||
active profile. | ||||
|
r26129 | |||
|
r4988 | Parameters | ||
---------- | ||||
profile : str | ||||
|
r27290 | The name of a profile which has a history file. | ||
|
r4988 | """ | ||
|
r26131 | return Path(locate_profile(profile)) / "history.sqlite" | ||
|
r26129 | |||
|
r8461 | @catch_corrupt_db | ||
|
r3388 | def init_db(self): | ||
|
r3405 | """Connect to the database, and create tables if necessary.""" | ||
|
r8176 | if not self.enabled: | ||
|
r8165 | self.db = DummyDB() | ||
return | ||||
|
r26129 | |||
|
r4487 | # use detect_types so that timestamps return datetime objects | ||
|
r8165 | kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) | ||
kwargs.update(self.connection_options) | ||||
|
r26130 | self.db = sqlite3.connect(str(self.hist_file), **kwargs) | ||
|
r27339 | with self.db: | ||
self.db.execute( | ||||
"""CREATE TABLE IF NOT EXISTS sessions (session integer | ||||
primary key autoincrement, start timestamp, | ||||
end timestamp, num_cmds integer, remark text)""" | ||||
) | ||||
self.db.execute( | ||||
"""CREATE TABLE IF NOT EXISTS history | ||||
(session integer, line integer, source text, source_raw text, | ||||
PRIMARY KEY (session, line))""" | ||||
) | ||||
# Output history is optional, but ensure the table's there so it can be | ||||
# enabled later. | ||||
self.db.execute( | ||||
"""CREATE TABLE IF NOT EXISTS output_history | ||||
(session integer, line integer, output text, | ||||
PRIMARY KEY (session, line))""" | ||||
) | ||||
|
r22270 | # success! reset corrupt db count | ||
self._corrupt_db_counter = 0 | ||||
|
r4872 | |||
|
r4876 | def writeout_cache(self): | ||
"""Overridden by HistoryManager to dump the cache before certain | ||||
database lookups.""" | ||||
pass | ||||
|
r3405 | ## ------------------------------- | ||
## Methods for retrieving history: | ||||
## ------------------------------- | ||||
|
r27045 | def _run_sql(self, sql, params, raw=True, output=False, latest=False): | ||
|
r3401 | """Prepares and runs an SQL query for the history database. | ||
|
r4872 | |||
|
r3401 | Parameters | ||
---------- | ||||
sql : str | ||||
|
r27290 | Any filtering expressions to go after SELECT ... FROM ... | ||
|
r3401 | params : tuple | ||
|
r27290 | Parameters passed to the SQL query (to replace "?") | ||
|
r3435 | raw, output : bool | ||
|
r27290 | See :meth:`get_range` | ||
|
r27045 | latest : bool | ||
|
r27290 | Select rows with max (session, line) | ||
|
r4872 | |||
|
r3401 | Returns | ||
------- | ||||
|
r3435 | Tuples as :meth:`get_range` | ||
|
r3401 | """ | ||
|
r3388 | toget = 'source_raw' if raw else 'source' | ||
|
r3397 | sqlfrom = "history" | ||
if output: | ||||
sqlfrom = "history LEFT JOIN output_history USING (session, line)" | ||||
toget = "history.%s, output_history.output" % toget | ||||
|
r27045 | if latest: | ||
toget += ", MAX(session * 128 * 1024 + line)" | ||||
|
r27646 | this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql | ||
cur = self.db.execute(this_querry, params) | ||||
|
r27045 | if latest: | ||
cur = (row[:-1] for row in cur) | ||||
|
r3415 | if output: # Regroup into 3-tuples, and parse JSON | ||
|
r3741 | return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) | ||
|
r3401 | return cur | ||
|
r4872 | |||
|
r25523 | @only_when_enabled | ||
|
r8461 | @catch_corrupt_db | ||
|
r16033 | def get_session_info(self, session): | ||
"""Get info about a session. | ||||
|
r4872 | |||
|
r4487 | Parameters | ||
---------- | ||||
session : int | ||||
|
r15979 | Session number to retrieve. | ||
|
r4872 | |||
|
r4487 | Returns | ||
------- | ||||
|
r15979 | session_id : int | ||
|
r27290 | Session ID number | ||
|
r16033 | start : datetime | ||
|
r27290 | Timestamp for the start of the session. | ||
|
r16033 | end : datetime | ||
|
r27290 | Timestamp for the end of the session, or None if IPython crashed. | ||
|
r16033 | num_cmds : int | ||
|
r27290 | Number of commands run, or None if IPython crashed. | ||
|
r16033 | remark : unicode | ||
|
r27290 | A manually set description. | ||
|
r4487 | """ | ||
query = "SELECT * from sessions where session == ?" | ||||
return self.db.execute(query, (session,)).fetchone() | ||||
|
r4872 | |||
|
r8463 | @catch_corrupt_db | ||
|
r15979 | def get_last_session_id(self): | ||
"""Get the last session ID currently in the database. | ||||
|
r26129 | |||
|
r15979 | Within IPython, this should be the same as the value stored in | ||
:attr:`HistoryManager.session_number`. | ||||
""" | ||||
for record in self.get_tail(n=1, include_latest=True): | ||||
return record[0] | ||||
@catch_corrupt_db | ||||
|
r3435 | def get_tail(self, n=10, raw=True, output=False, include_latest=False): | ||
|
r3420 | """Get the last n lines from the history database. | ||
|
r4872 | |||
|
r3435 | Parameters | ||
---------- | ||||
n : int | ||||
|
r27290 | The number of lines to get | ||
|
r3435 | raw, output : bool | ||
|
r27290 | See :meth:`get_range` | ||
|
r3435 | include_latest : bool | ||
|
r27290 | If False (default), n+1 lines are fetched, and the latest one | ||
is discarded. This is intended to be used where the function | ||||
is called by a user command, which it should not return. | ||||
|
r4872 | |||
|
r3435 | Returns | ||
------- | ||||
Tuples as :meth:`get_range` | ||||
""" | ||||
|
r3403 | self.writeout_cache() | ||
|
r3420 | if not include_latest: | ||
n += 1 | ||||
|
r27708 | cur = self._run_sql( | ||
"ORDER BY session DESC, line DESC LIMIT ?", (n,), raw=raw, output=output | ||||
) | ||||
|
r3420 | if not include_latest: | ||
|
r27707 | return reversed(list(cur)[1:]) | ||
return reversed(list(cur)) | ||||
|
r4872 | |||
|
r8463 | @catch_corrupt_db | ||
|
r3435 | def search(self, pattern="*", raw=True, search_raw=True, | ||
|
r8780 | output=False, n=None, unique=False): | ||
|
r3420 | """Search the database using unix glob-style matching (wildcards | ||
* and ?). | ||||
|
r4872 | |||
|
r3435 | Parameters | ||
---------- | ||||
pattern : str | ||||
|
r27290 | The wildcarded pattern to match when searching | ||
|
r3435 | search_raw : bool | ||
|
r27290 | If True, search the raw input, otherwise, the parsed input | ||
|
r3435 | raw, output : bool | ||
|
r27290 | See :meth:`get_range` | ||
|
r8399 | n : None or int | ||
|
r27290 | If an integer is given, it defines the limit of | ||
returned entries. | ||||
|
r8780 | unique : bool | ||
|
r27290 | When it is true, return only unique entries. | ||
|
r4872 | |||
|
r3388 | Returns | ||
------- | ||||
|
r3435 | Tuples as :meth:`get_range` | ||
|
r3388 | """ | ||
|
r3418 | tosearch = "source_raw" if search_raw else "source" | ||
|
r3400 | if output: | ||
tosearch = "history." + tosearch | ||||
|
r3403 | self.writeout_cache() | ||
|
r8399 | sqlform = "WHERE %s GLOB ?" % tosearch | ||
params = (pattern,) | ||||
|
r8780 | if unique: | ||
sqlform += ' GROUP BY {0}'.format(tosearch) | ||||
|
r8399 | if n is not None: | ||
sqlform += " ORDER BY session DESC, line DESC LIMIT ?" | ||||
params += (n,) | ||||
|
r8780 | elif unique: | ||
sqlform += " ORDER BY session, line" | ||||
|
r27045 | cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique) | ||
|
r8399 | if n is not None: | ||
|
r8403 | return reversed(list(cur)) | ||
|
r8399 | return cur | ||
|
r26129 | |||
|
r8463 | @catch_corrupt_db | ||
|
r4876 | def get_range(self, session, start=1, stop=None, raw=True,output=False): | ||
|
r3394 | """Retrieve input by session. | ||
|
r4872 | |||
|
r3394 | Parameters | ||
---------- | ||||
session : int | ||||
|
r4876 | Session number to retrieve. | ||
|
r3394 | start : int | ||
First line to retrieve. | ||||
stop : int | ||||
|
r3396 | End of line range (excluded from output itself). If None, retrieve | ||
to the end of the session. | ||||
|
r3394 | raw : bool | ||
If True, return untranslated input | ||||
output : bool | ||||
If True, attempt to include output. This will be 'real' Python | ||||
objects for the current session, or text reprs from previous | ||||
sessions if db_log_output was enabled at the time. Where no output | ||||
is found, None is used. | ||||
|
r4872 | |||
|
r3394 | Returns | ||
------- | ||||
|
r16033 | entries | ||
|
r27290 | An iterator over the desired lines. Each line is a 3-tuple, either | ||
(session, line, input) if output is False, or | ||||
(session, line, (input, output)) if output is True. | ||||
|
r3394 | """ | ||
if stop: | ||||
|
r3395 | lineclause = "line >= ? AND line < ?" | ||
|
r3394 | params = (session, start, stop) | ||
else: | ||||
lineclause = "line>=?" | ||||
params = (session, start) | ||||
|
r4872 | |||
|
r8397 | return self._run_sql("WHERE session==? AND %s" % lineclause, | ||
|
r3401 | params, raw=raw, output=output) | ||
|
r4872 | |||
|
r3435 | def get_range_by_str(self, rangestr, raw=True, output=False): | ||
|
r3394 | """Get lines of history from a string of ranges, as used by magic | ||
|
r3435 | commands %hist, %save, %macro, etc. | ||
|
r4872 | |||
|
r3435 | Parameters | ||
---------- | ||||
rangestr : str | ||||
|
r27290 | A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used, | ||
this will return everything from current session's history. | ||||
|
r26630 | |||
|
r27290 | See the documentation of :func:`%history` for the full details. | ||
|
r26630 | |||
|
r3435 | raw, output : bool | ||
|
r27290 | As :meth:`get_range` | ||
|
r4872 | |||
|
r3435 | Returns | ||
------- | ||||
Tuples as :meth:`get_range` | ||||
""" | ||||
|
r3395 | for sess, s, e in extract_hist_ranges(rangestr): | ||
|
r3435 | for line in self.get_range(sess, s, e, raw=raw, output=output): | ||
|
r3394 | yield line | ||
|
r4872 | |||
|
r4875 | |||
class HistoryManager(HistoryAccessor): | ||||
"""A class to organize all history-related functionality in one place. | ||||
""" | ||||
# Public interface | ||||
# An instance of the IPython shell we are attached to | ||||
|
r20940 | shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', | ||
allow_none=True) | ||||
|
r4875 | # Lists to hold processed and raw history. These start with a blank entry | ||
# so that we can index them starting from 1 | ||||
input_hist_parsed = List([""]) | ||||
input_hist_raw = List([""]) | ||||
# A list of directories visited during session | ||||
dir_hist = List() | ||||
|
r22340 | @default('dir_hist') | ||
|
r4875 | def _dir_hist_default(self): | ||
try: | ||||
|
r26129 | return [Path.cwd()] | ||
|
r4875 | except OSError: | ||
return [] | ||||
# A dict of output history, keyed with ints from the shell's | ||||
# execution count. | ||||
output_hist = Dict() | ||||
# The text/plain repr of outputs. | ||||
output_hist_reprs = Dict() | ||||
# The number of the current session in the history database | ||||
|
r5344 | session_number = Integer() | ||
|
r26129 | |||
|
r22340 | db_log_output = Bool(False, | ||
|
r13804 | help="Should the history database include output? (default: no)" | ||
|
r22340 | ).tag(config=True) | ||
db_cache_size = Integer(0, | ||||
|
r13804 | help="Write to database every x commands (higher values save disk access & power).\n" | ||
"Values of 1 or less effectively disable caching." | ||||
|
r22340 | ).tag(config=True) | ||
|
r4875 | # The input and output caches | ||
db_input_cache = List() | ||||
db_output_cache = List() | ||||
|
r26129 | |||
|
r4875 | # History saving in separate thread | ||
|
r20940 | save_thread = Instance('IPython.core.history.HistorySavingThread', | ||
allow_none=True) | ||||
|
r23468 | save_flag = Instance(threading.Event, allow_none=True) | ||
|
r26129 | |||
|
r4875 | # Private interface | ||
# Variables used to store the three last inputs from the user. On each new | ||||
# history update, we populate the user's namespace with these, shifted as | ||||
# necessary. | ||||
_i00 = Unicode(u'') | ||||
_i = Unicode(u'') | ||||
_ii = Unicode(u'') | ||||
_iii = Unicode(u'') | ||||
# A regex matching all forms of the exit command, so that we don't store | ||||
# them in the history (it's annoying to rewind the first entry and land on | ||||
# an exit call). | ||||
_exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") | ||||
def __init__(self, shell=None, config=None, **traits): | ||||
"""Create a new history manager associated with a shell instance. | ||||
""" | ||||
super(HistoryManager, self).__init__(shell=shell, config=config, | ||||
**traits) | ||||
self.save_flag = threading.Event() | ||||
self.db_input_cache_lock = threading.Lock() | ||||
self.db_output_cache_lock = threading.Lock() | ||||
|
r26129 | |||
|
r21697 | try: | ||
self.new_session() | ||||
|
r25522 | except sqlite3.OperationalError: | ||
|
r21697 | self.log.error("Failed to create history session in %s. History will not be saved.", | ||
self.hist_file, exc_info=True) | ||||
self.hist_file = ':memory:' | ||||
|
r26129 | |||
|
r8176 | if self.enabled and self.hist_file != ':memory:': | ||
|
r6871 | self.save_thread = HistorySavingThread(self) | ||
self.save_thread.start() | ||||
|
r4875 | |||
|
r4988 | def _get_hist_file_name(self, profile=None): | ||
"""Get default history file name based on the Shell's profile. | ||||
|
r26129 | |||
|
r4988 | The profile parameter is ignored, but must exist for compatibility with | ||
the parent class.""" | ||||
|
r4875 | profile_dir = self.shell.profile_dir.location | ||
|
r26131 | return Path(profile_dir) / "history.sqlite" | ||
|
r26129 | |||
|
r25523 | @only_when_enabled | ||
|
r4875 | def new_session(self, conn=None): | ||
"""Get a new session number.""" | ||||
if conn is None: | ||||
conn = self.db | ||||
|
r26129 | |||
|
r4875 | with conn: | ||
|
r28159 | cur = conn.execute( | ||
"""INSERT INTO sessions VALUES (NULL, ?, NULL, | ||||
NULL, '') """, | ||||
(datetime.datetime.now(),), | ||||
) | ||||
|
r4875 | self.session_number = cur.lastrowid | ||
|
r26129 | |||
|
r4875 | def end_session(self): | ||
"""Close the database session, filling in the end time and line count.""" | ||||
self.writeout_cache() | ||||
with self.db: | ||||
self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE | ||||
session==?""", (datetime.datetime.now(), | ||||
len(self.input_hist_parsed)-1, self.session_number)) | ||||
self.session_number = 0 | ||||
|
r26129 | |||
|
r4875 | def name_session(self, name): | ||
"""Give the current session a name in the history database.""" | ||||
with self.db: | ||||
self.db.execute("UPDATE sessions SET remark=? WHERE session==?", | ||||
(name, self.session_number)) | ||||
|
r26129 | |||
|
r4875 | def reset(self, new_session=True): | ||
"""Clear the session history, releasing all object references, and | ||||
optionally open a new session.""" | ||||
self.output_hist.clear() | ||||
# The directory history can't be completely empty | ||||
|
r26129 | self.dir_hist[:] = [Path.cwd()] | ||
|
r4875 | if new_session: | ||
if self.session_number: | ||||
self.end_session() | ||||
self.input_hist_parsed[:] = [""] | ||||
self.input_hist_raw[:] = [""] | ||||
self.new_session() | ||||
|
r18815 | |||
|
r4875 | # ------------------------------ | ||
# Methods for retrieving history | ||||
# ------------------------------ | ||||
|
r15979 | def get_session_info(self, session=0): | ||
|
r16033 | """Get info about a session. | ||
|
r15979 | |||
Parameters | ||||
---------- | ||||
session : int | ||||
Session number to retrieve. The current session is 0, and negative | ||||
|
r16033 | numbers count back from current session, so -1 is the previous session. | ||
|
r15979 | |||
Returns | ||||
------- | ||||
session_id : int | ||||
|
r27290 | Session ID number | ||
|
r16033 | start : datetime | ||
|
r27290 | Timestamp for the start of the session. | ||
|
r16033 | end : datetime | ||
|
r27290 | Timestamp for the end of the session, or None if IPython crashed. | ||
|
r16033 | num_cmds : int | ||
|
r27290 | Number of commands run, or None if IPython crashed. | ||
|
r16033 | remark : unicode | ||
|
r27290 | A manually set description. | ||
|
r15979 | """ | ||
if session <= 0: | ||||
session += self.session_number | ||||
return super(HistoryManager, self).get_session_info(session=session) | ||||
|
r27707 | @catch_corrupt_db | ||
def get_tail(self, n=10, raw=True, output=False, include_latest=False): | ||||
"""Get the last n lines from the history database. | ||||
Most recent entry last. | ||||
Completion will be reordered so that that the last ones are when | ||||
possible from current session. | ||||
Parameters | ||||
---------- | ||||
n : int | ||||
The number of lines to get | ||||
raw, output : bool | ||||
See :meth:`get_range` | ||||
include_latest : bool | ||||
If False (default), n+1 lines are fetched, and the latest one | ||||
is discarded. This is intended to be used where the function | ||||
is called by a user command, which it should not return. | ||||
Returns | ||||
------- | ||||
Tuples as :meth:`get_range` | ||||
""" | ||||
self.writeout_cache() | ||||
if not include_latest: | ||||
n += 1 | ||||
# cursor/line/entry | ||||
this_cur = list( | ||||
self._run_sql( | ||||
"WHERE session == ? ORDER BY line DESC LIMIT ? ", | ||||
(self.session_number, n), | ||||
raw=raw, | ||||
output=output, | ||||
) | ||||
) | ||||
other_cur = list( | ||||
self._run_sql( | ||||
"WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?", | ||||
(self.session_number, n), | ||||
raw=raw, | ||||
output=output, | ||||
) | ||||
) | ||||
everything = this_cur + other_cur | ||||
everything = everything[:n] | ||||
if not include_latest: | ||||
return list(everything)[:0:-1] | ||||
return list(everything)[::-1] | ||||
|
r4875 | def _get_range_session(self, start=1, stop=None, raw=True, output=False): | ||
"""Get input and output history from the current session. Called by | ||||
get_range, and takes similar parameters.""" | ||||
input_hist = self.input_hist_raw if raw else self.input_hist_parsed | ||||
|
r26129 | |||
|
r4875 | n = len(input_hist) | ||
if start < 0: | ||||
start += n | ||||
if not stop or (stop > n): | ||||
stop = n | ||||
elif stop < 0: | ||||
stop += n | ||||
|
r26129 | |||
|
r4875 | for i in range(start, stop): | ||
if output: | ||||
line = (input_hist[i], self.output_hist_reprs.get(i)) | ||||
else: | ||||
line = input_hist[i] | ||||
yield (0, i, line) | ||||
|
r26129 | |||
|
r4877 | def get_range(self, session=0, start=1, stop=None, raw=True,output=False): | ||
|
r4876 | """Retrieve input by session. | ||
|
r26129 | |||
|
r4876 | Parameters | ||
---------- | ||||
session : int | ||||
Session number to retrieve. The current session is 0, and negative | ||||
numbers count back from current session, so -1 is previous session. | ||||
start : int | ||||
First line to retrieve. | ||||
stop : int | ||||
End of line range (excluded from output itself). If None, retrieve | ||||
to the end of the session. | ||||
raw : bool | ||||
If True, return untranslated input | ||||
output : bool | ||||
If True, attempt to include output. This will be 'real' Python | ||||
objects for the current session, or text reprs from previous | ||||
sessions if db_log_output was enabled at the time. Where no output | ||||
is found, None is used. | ||||
|
r26129 | |||
|
r4876 | Returns | ||
------- | ||||
|
r16033 | entries | ||
|
r27290 | An iterator over the desired lines. Each line is a 3-tuple, either | ||
(session, line, input) if output is False, or | ||||
(session, line, (input, output)) if output is True. | ||||
|
r4876 | """ | ||
if session <= 0: | ||||
session += self.session_number | ||||
if session==self.session_number: # Current session | ||||
return self._get_range_session(start, stop, raw, output) | ||||
|
r6932 | return super(HistoryManager, self).get_range(session, start, stop, raw, | ||
output) | ||||
|
r4875 | |||
|
r3405 | ## ---------------------------- | ||
## Methods for storing history: | ||||
## ---------------------------- | ||||
|
r3390 | def store_inputs(self, line_num, source, source_raw=None): | ||
|
r3087 | """Store source and raw input in history and create input cache | ||
|
r16033 | variables ``_i*``. | ||
|
r4872 | |||
|
r3080 | Parameters | ||
---------- | ||||
|
r3390 | line_num : int | ||
|
r27290 | The prompt number of this input. | ||
|
r3080 | source : str | ||
|
r27290 | Python input. | ||
|
r3080 | source_raw : str, optional | ||
|
r27290 | If given, this is the raw input without any IPython transformations | ||
applied to it. If not given, ``source`` is used. | ||||
|
r3080 | """ | ||
if source_raw is None: | ||||
source_raw = source | ||||
|
r3419 | source = source.rstrip('\n') | ||
source_raw = source_raw.rstrip('\n') | ||||
|
r4872 | |||
|
r3246 | # do not store exit/quit commands | ||
|
r3750 | if self._exit_re.match(source_raw.strip()): | ||
|
r3243 | return | ||
|
r4872 | |||
|
r3418 | self.input_hist_parsed.append(source) | ||
self.input_hist_raw.append(source_raw) | ||||
|
r4872 | |||
|
r3713 | with self.db_input_cache_lock: | ||
self.db_input_cache.append((line_num, source, source_raw)) | ||||
# Trigger to flush cache and write to DB. | ||||
if len(self.db_input_cache) >= self.db_cache_size: | ||||
self.save_flag.set() | ||||
|
r3080 | |||
|
r3087 | # update the auto _i variables | ||
self._iii = self._ii | ||||
self._ii = self._i | ||||
self._i = self._i00 | ||||
self._i00 = source_raw | ||||
# hackish access to user namespace to create _i1,_i2... dynamically | ||||
|
r3390 | new_i = '_i%s' % line_num | ||
|
r3087 | to_main = {'_i': self._i, | ||
'_ii': self._ii, | ||||
'_iii': self._iii, | ||||
new_i : self._i00 } | ||||
|
r26129 | |||
|
r11359 | if self.shell is not None: | ||
self.shell.push(to_main, interactive=False) | ||||
|
r4872 | |||
|
r3415 | def store_output(self, line_num): | ||
|
r3417 | """If database output logging is enabled, this saves all the | ||
outputs from the indicated prompt number to the database. It's | ||||
|
r3435 | called by run_cell after code has been executed. | ||
|
r4872 | |||
|
r3435 | Parameters | ||
---------- | ||||
line_num : int | ||||
|
r27290 | The line number from which to save outputs | ||
|
r3435 | """ | ||
|
r3741 | if (not self.db_log_output) or (line_num not in self.output_hist_reprs): | ||
|
r3391 | return | ||
|
r3741 | output = self.output_hist_reprs[line_num] | ||
|
r4872 | |||
|
r3713 | with self.db_output_cache_lock: | ||
self.db_output_cache.append((line_num, output)) | ||||
|
r3437 | if self.db_cache_size <= 1: | ||
|
r3711 | self.save_flag.set() | ||
|
r4872 | |||
|
r3711 | def _writeout_input_cache(self, conn): | ||
with conn: | ||||
|
r3700 | for line in self.db_input_cache: | ||
|
r3711 | conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)", | ||
|
r3437 | (self.session_number,)+line) | ||
|
r4872 | |||
|
r3711 | def _writeout_output_cache(self, conn): | ||
with conn: | ||||
|
r3700 | for line in self.db_output_cache: | ||
|
r3711 | conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", | ||
|
r3437 | (self.session_number,)+line) | ||
|
r4872 | |||
|
r25523 | @only_when_enabled | ||
|
r3711 | def writeout_cache(self, conn=None): | ||
|
r3437 | """Write any entries in the cache to the database.""" | ||
|
r3711 | if conn is None: | ||
conn = self.db | ||||
|
r4872 | |||
|
r3713 | with self.db_input_cache_lock: | ||
try: | ||||
|
r3711 | self._writeout_input_cache(conn) | ||
|
r3437 | except sqlite3.IntegrityError: | ||
|
r3714 | self.new_session(conn) | ||
|
r3713 | print("ERROR! Session/line number was not unique in", | ||
"database. History logging moved to new session", | ||||
self.session_number) | ||||
|
r6932 | try: | ||
# Try writing to the new session. If this fails, don't | ||||
# recurse | ||||
|
r3713 | self._writeout_input_cache(conn) | ||
except sqlite3.IntegrityError: | ||||
pass | ||||
finally: | ||||
self.db_input_cache = [] | ||||
with self.db_output_cache_lock: | ||||
try: | ||||
self._writeout_output_cache(conn) | ||||
except sqlite3.IntegrityError: | ||||
print("!! Session/line number for output was not unique", | ||||
"in database. Output will not be stored.") | ||||
finally: | ||||
self.db_output_cache = [] | ||||
|
r3087 | |||
|
r3711 | |||
class HistorySavingThread(threading.Thread): | ||||
|
r3716 | """This thread takes care of writing history to the database, so that | ||
the UI isn't held up while that happens. | ||||
|
r4872 | |||
|
r3716 | It waits for the HistoryManager's save_flag to be set, then writes out | ||
the history cache. The main thread is responsible for setting the flag when | ||||
the cache size reaches a defined threshold.""" | ||||
|
r3711 | daemon = True | ||
stop_now = False | ||||
|
r8176 | enabled = True | ||
|
r3711 | def __init__(self, history_manager): | ||
|
r13126 | super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread") | ||
|
r3711 | self.history_manager = history_manager | ||
|
r8176 | self.enabled = history_manager.enabled | ||
|
r3711 | atexit.register(self.stop) | ||
|
r4872 | |||
|
r25523 | @only_when_enabled | ||
|
r3711 | def run(self): | ||
# We need a separate db connection per thread: | ||||
|
r3720 | try: | ||
|
r26131 | self.db = sqlite3.connect( | ||
str(self.history_manager.hist_file), | ||||
|
r27045 | **self.history_manager.connection_options, | ||
|
r8165 | ) | ||
|
r3720 | while True: | ||
self.history_manager.save_flag.wait() | ||||
if self.stop_now: | ||||
|
r16775 | self.db.close() | ||
|
r3720 | return | ||
self.history_manager.save_flag.clear() | ||||
self.history_manager.writeout_cache(self.db) | ||||
except Exception as e: | ||||
print(("The history saving thread hit an unexpected error (%s)." | ||||
"History will not be written to the database.") % repr(e)) | ||||
|
r4872 | |||
|
r3711 | def stop(self): | ||
|
r3716 | """This can be called from the main thread to safely stop this thread. | ||
|
r4872 | |||
|
r3716 | Note that it does not attempt to write out remaining history before | ||
exiting. That should be done by calling the HistoryManager's | ||||
end_session method.""" | ||||
|
r3711 | self.stop_now = True | ||
self.history_manager.save_flag.set() | ||||
|
r3716 | self.join() | ||
|
r3711 | |||
|
r4872 | |||
|
r3395 | # To match, e.g. ~5/8-~2/3 | ||
|
r3394 | range_re = re.compile(r""" | ||
|
r3395 | ((?P<startsess>~?\d+)/)? | ||
|
r9786 | (?P<start>\d+)? | ||
|
r3394 | ((?P<sep>[\-:]) | ||
|
r3395 | ((?P<endsess>~?\d+)/)? | ||
|
r3394 | (?P<end>\d+))? | ||
|
r3493 | $""", re.VERBOSE) | ||
|
r3394 | |||
|
r6911 | |||
|
r3394 | def extract_hist_ranges(ranges_str): | ||
"""Turn a string of history ranges into 3-tuples of (session, start, stop). | ||||
|
r4872 | |||
|
r26630 | Empty string results in a `[(0, 1, None)]`, i.e. "everything from current | ||
session". | ||||
|
r3394 | Examples | ||
-------- | ||||
|
r16036 | >>> list(extract_hist_ranges("~8/5-~7/4 2")) | ||
[(-8, 5, None), (-7, 1, 5), (0, 2, 3)] | ||||
|
r3394 | """ | ||
|
r26641 | if ranges_str == "": | ||
|
r26630 | yield (0, 1, None) # Everything from current session | ||
return | ||||
|
r3394 | for range_str in ranges_str.split(): | ||
rmatch = range_re.match(range_str) | ||||
|
r3416 | if not rmatch: | ||
continue | ||||
|
r9786 | start = rmatch.group("start") | ||
if start: | ||||
start = int(start) | ||||
end = rmatch.group("end") | ||||
# If no end specified, get (a, a + 1) | ||||
end = int(end) if end else start + 1 | ||||
else: # start not specified | ||||
if not rmatch.group('startsess'): # no startsess | ||||
continue | ||||
start = 1 | ||||
end = None # provide the entire session hist | ||||
|
r3394 | if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3] | ||
end += 1 | ||||
startsess = rmatch.group("startsess") or "0" | ||||
endsess = rmatch.group("endsess") or startsess | ||||
startsess = int(startsess.replace("~","-")) | ||||
endsess = int(endsess.replace("~","-")) | ||||
|
r9791 | assert endsess >= startsess, "start session must be earlier than end session" | ||
|
r3394 | |||
if endsess == startsess: | ||||
yield (startsess, start, end) | ||||
continue | ||||
# Multiple sessions in one range: | ||||
yield (startsess, start, None) | ||||
for sess in range(startsess+1, endsess): | ||||
yield (sess, 1, None) | ||||
yield (endsess, 1, end) | ||||
|
r6911 | |||
|
r3394 | def _format_lineno(session, line): | ||
"""Helper function to format line numbers properly.""" | ||||
if session == 0: | ||||
return str(line) | ||||
return "%s#%s" % (session, line) | ||||