##// END OF EJS Templates
Merge pull request #9687 from Carreau/memory-history...
Thomas Kluyver -
r22637:ce175464 merge
parent child Browse files
Show More
@@ -1,908 +1,911 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 from __future__ import print_function
6 from __future__ import print_function
7
7
8 import atexit
8 import atexit
9 import datetime
9 import datetime
10 import os
10 import os
11 import re
11 import re
12 try:
12 try:
13 import sqlite3
13 import sqlite3
14 except ImportError:
14 except ImportError:
15 try:
15 try:
16 from pysqlite2 import dbapi2 as sqlite3
16 from pysqlite2 import dbapi2 as sqlite3
17 except ImportError:
17 except ImportError:
18 sqlite3 = None
18 sqlite3 = None
19 import threading
19 import threading
20
20
21 from traitlets.config.configurable import LoggingConfigurable
21 from traitlets.config.configurable import LoggingConfigurable
22 from decorator import decorator
22 from decorator import decorator
23 from IPython.utils.decorators import undoc
23 from IPython.utils.decorators import undoc
24 from IPython.utils.path import locate_profile
24 from IPython.utils.path import locate_profile
25 from IPython.utils import py3compat
25 from IPython.utils import py3compat
26 from traitlets import (
26 from traitlets import (
27 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
27 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
28 default, observe,
28 default, observe,
29 )
29 )
30 from warnings import warn
30 from warnings import warn
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Classes and functions
33 # Classes and functions
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 @undoc
36 @undoc
37 class DummyDB(object):
37 class DummyDB(object):
38 """Dummy DB that will act as a black hole for history.
38 """Dummy DB that will act as a black hole for history.
39
39
40 Only used in the absence of sqlite"""
40 Only used in the absence of sqlite"""
41 def execute(*args, **kwargs):
41 def execute(*args, **kwargs):
42 return []
42 return []
43
43
44 def commit(self, *args, **kwargs):
44 def commit(self, *args, **kwargs):
45 pass
45 pass
46
46
47 def __enter__(self, *args, **kwargs):
47 def __enter__(self, *args, **kwargs):
48 pass
48 pass
49
49
50 def __exit__(self, *args, **kwargs):
50 def __exit__(self, *args, **kwargs):
51 pass
51 pass
52
52
53
53
54 @decorator
54 @decorator
55 def needs_sqlite(f, self, *a, **kw):
55 def needs_sqlite(f, self, *a, **kw):
56 """Decorator: return an empty list in the absence of sqlite."""
56 """Decorator: return an empty list in the absence of sqlite."""
57 if sqlite3 is None or not self.enabled:
57 if sqlite3 is None or not self.enabled:
58 return []
58 return []
59 else:
59 else:
60 return f(self, *a, **kw)
60 return f(self, *a, **kw)
61
61
62
62
63 if sqlite3 is not None:
63 if sqlite3 is not None:
64 DatabaseError = sqlite3.DatabaseError
64 DatabaseError = sqlite3.DatabaseError
65 OperationalError = sqlite3.OperationalError
65 OperationalError = sqlite3.OperationalError
66 else:
66 else:
67 @undoc
67 @undoc
68 class DatabaseError(Exception):
68 class DatabaseError(Exception):
69 "Dummy exception when sqlite could not be imported. Should never occur."
69 "Dummy exception when sqlite could not be imported. Should never occur."
70
70
71 @undoc
71 @undoc
72 class OperationalError(Exception):
72 class OperationalError(Exception):
73 "Dummy exception when sqlite could not be imported. Should never occur."
73 "Dummy exception when sqlite could not be imported. Should never occur."
74
74
75 # use 16kB as threshold for whether a corrupt history db should be saved
75 # use 16kB as threshold for whether a corrupt history db should be saved
76 # that should be at least 100 entries or so
76 # that should be at least 100 entries or so
77 _SAVE_DB_SIZE = 16384
77 _SAVE_DB_SIZE = 16384
78
78
79 @decorator
79 @decorator
80 def catch_corrupt_db(f, self, *a, **kw):
80 def catch_corrupt_db(f, self, *a, **kw):
81 """A decorator which wraps HistoryAccessor method calls to catch errors from
81 """A decorator which wraps HistoryAccessor method calls to catch errors from
82 a corrupt SQLite database, move the old database out of the way, and create
82 a corrupt SQLite database, move the old database out of the way, and create
83 a new one.
83 a new one.
84
84
85 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
85 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
86 not just a corrupt file.
86 not just a corrupt file.
87 """
87 """
88 try:
88 try:
89 return f(self, *a, **kw)
89 return f(self, *a, **kw)
90 except (DatabaseError, OperationalError) as e:
90 except (DatabaseError, OperationalError) as e:
91 self._corrupt_db_counter += 1
91 self._corrupt_db_counter += 1
92 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
92 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
93 if self.hist_file != ':memory:':
93 if self.hist_file != ':memory:':
94 if self._corrupt_db_counter > self._corrupt_db_limit:
94 if self._corrupt_db_counter > self._corrupt_db_limit:
95 self.hist_file = ':memory:'
95 self.hist_file = ':memory:'
96 self.log.error("Failed to load history too many times, history will not be saved.")
96 self.log.error("Failed to load history too many times, history will not be saved.")
97 elif os.path.isfile(self.hist_file):
97 elif os.path.isfile(self.hist_file):
98 # move the file out of the way
98 # move the file out of the way
99 base, ext = os.path.splitext(self.hist_file)
99 base, ext = os.path.splitext(self.hist_file)
100 size = os.stat(self.hist_file).st_size
100 size = os.stat(self.hist_file).st_size
101 if size >= _SAVE_DB_SIZE:
101 if size >= _SAVE_DB_SIZE:
102 # if there's significant content, avoid clobbering
102 # if there's significant content, avoid clobbering
103 now = datetime.datetime.now().isoformat().replace(':', '.')
103 now = datetime.datetime.now().isoformat().replace(':', '.')
104 newpath = base + '-corrupt-' + now + ext
104 newpath = base + '-corrupt-' + now + ext
105 # don't clobber previous corrupt backups
105 # don't clobber previous corrupt backups
106 for i in range(100):
106 for i in range(100):
107 if not os.path.isfile(newpath):
107 if not os.path.isfile(newpath):
108 break
108 break
109 else:
109 else:
110 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
110 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
111 else:
111 else:
112 # not much content, possibly empty; don't worry about clobbering
112 # not much content, possibly empty; don't worry about clobbering
113 # maybe we should just delete it?
113 # maybe we should just delete it?
114 newpath = base + '-corrupt' + ext
114 newpath = base + '-corrupt' + ext
115 os.rename(self.hist_file, newpath)
115 os.rename(self.hist_file, newpath)
116 self.log.error("History file was moved to %s and a new file created.", newpath)
116 self.log.error("History file was moved to %s and a new file created.", newpath)
117 self.init_db()
117 self.init_db()
118 return []
118 return []
119 else:
119 else:
120 # Failed with :memory:, something serious is wrong
120 # Failed with :memory:, something serious is wrong
121 raise
121 raise
122
122
123 class HistoryAccessorBase(LoggingConfigurable):
123 class HistoryAccessorBase(LoggingConfigurable):
124 """An abstract class for History Accessors """
124 """An abstract class for History Accessors """
125
125
126 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
126 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
127 raise NotImplementedError
127 raise NotImplementedError
128
128
129 def search(self, pattern="*", raw=True, search_raw=True,
129 def search(self, pattern="*", raw=True, search_raw=True,
130 output=False, n=None, unique=False):
130 output=False, n=None, unique=False):
131 raise NotImplementedError
131 raise NotImplementedError
132
132
133 def get_range(self, session, start=1, stop=None, raw=True,output=False):
133 def get_range(self, session, start=1, stop=None, raw=True,output=False):
134 raise NotImplementedError
134 raise NotImplementedError
135
135
136 def get_range_by_str(self, rangestr, raw=True, output=False):
136 def get_range_by_str(self, rangestr, raw=True, output=False):
137 raise NotImplementedError
137 raise NotImplementedError
138
138
139
139
140 class HistoryAccessor(HistoryAccessorBase):
140 class HistoryAccessor(HistoryAccessorBase):
141 """Access the history database without adding to it.
141 """Access the history database without adding to it.
142
142
143 This is intended for use by standalone history tools. IPython shells use
143 This is intended for use by standalone history tools. IPython shells use
144 HistoryManager, below, which is a subclass of this."""
144 HistoryManager, below, which is a subclass of this."""
145
145
146 # counter for init_db retries, so we don't keep trying over and over
146 # counter for init_db retries, so we don't keep trying over and over
147 _corrupt_db_counter = 0
147 _corrupt_db_counter = 0
148 # after two failures, fallback on :memory:
148 # after two failures, fallback on :memory:
149 _corrupt_db_limit = 2
149 _corrupt_db_limit = 2
150
150
151 # String holding the path to the history file
151 # String holding the path to the history file
152 hist_file = Unicode(
152 hist_file = Unicode(
153 help="""Path to file to use for SQLite history database.
153 help="""Path to file to use for SQLite history database.
154
154
155 By default, IPython will put the history database in the IPython
155 By default, IPython will put the history database in the IPython
156 profile directory. If you would rather share one history among
156 profile directory. If you would rather share one history among
157 profiles, you can set this value in each, so that they are consistent.
157 profiles, you can set this value in each, so that they are consistent.
158
158
159 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
159 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
160 mounts. If you see IPython hanging, try setting this to something on a
160 mounts. If you see IPython hanging, try setting this to something on a
161 local disk, e.g::
161 local disk, e.g::
162
162
163 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
163 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
164
164
165 you can also use the specific value `:memory:` (including the colon
166 at both end but not the back ticks), to avoid creating an history file.
167
165 """).tag(config=True)
168 """).tag(config=True)
166
169
167 enabled = Bool(True,
170 enabled = Bool(True,
168 help="""enable the SQLite history
171 help="""enable the SQLite history
169
172
170 set enabled=False to disable the SQLite history,
173 set enabled=False to disable the SQLite history,
171 in which case there will be no stored history, no SQLite connection,
174 in which case there will be no stored history, no SQLite connection,
172 and no background saving thread. This may be necessary in some
175 and no background saving thread. This may be necessary in some
173 threaded environments where IPython is embedded.
176 threaded environments where IPython is embedded.
174 """
177 """
175 ).tag(config=True)
178 ).tag(config=True)
176
179
177 connection_options = Dict(
180 connection_options = Dict(
178 help="""Options for configuring the SQLite connection
181 help="""Options for configuring the SQLite connection
179
182
180 These options are passed as keyword args to sqlite3.connect
183 These options are passed as keyword args to sqlite3.connect
181 when establishing database conenctions.
184 when establishing database conenctions.
182 """
185 """
183 ).tag(config=True)
186 ).tag(config=True)
184
187
185 # The SQLite database
188 # The SQLite database
186 db = Any()
189 db = Any()
187 @observe('db')
190 @observe('db')
188 def _db_changed(self, change):
191 def _db_changed(self, change):
189 """validate the db, since it can be an Instance of two different types"""
192 """validate the db, since it can be an Instance of two different types"""
190 new = change['new']
193 new = change['new']
191 connection_types = (DummyDB,)
194 connection_types = (DummyDB,)
192 if sqlite3 is not None:
195 if sqlite3 is not None:
193 connection_types = (DummyDB, sqlite3.Connection)
196 connection_types = (DummyDB, sqlite3.Connection)
194 if not isinstance(new, connection_types):
197 if not isinstance(new, connection_types):
195 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
198 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
196 (self.__class__.__name__, new)
199 (self.__class__.__name__, new)
197 raise TraitError(msg)
200 raise TraitError(msg)
198
201
199 def __init__(self, profile='default', hist_file=u'', **traits):
202 def __init__(self, profile='default', hist_file=u'', **traits):
200 """Create a new history accessor.
203 """Create a new history accessor.
201
204
202 Parameters
205 Parameters
203 ----------
206 ----------
204 profile : str
207 profile : str
205 The name of the profile from which to open history.
208 The name of the profile from which to open history.
206 hist_file : str
209 hist_file : str
207 Path to an SQLite history database stored by IPython. If specified,
210 Path to an SQLite history database stored by IPython. If specified,
208 hist_file overrides profile.
211 hist_file overrides profile.
209 config : :class:`~traitlets.config.loader.Config`
212 config : :class:`~traitlets.config.loader.Config`
210 Config object. hist_file can also be set through this.
213 Config object. hist_file can also be set through this.
211 """
214 """
212 # We need a pointer back to the shell for various tasks.
215 # We need a pointer back to the shell for various tasks.
213 super(HistoryAccessor, self).__init__(**traits)
216 super(HistoryAccessor, self).__init__(**traits)
214 # defer setting hist_file from kwarg until after init,
217 # defer setting hist_file from kwarg until after init,
215 # otherwise the default kwarg value would clobber any value
218 # otherwise the default kwarg value would clobber any value
216 # set by config
219 # set by config
217 if hist_file:
220 if hist_file:
218 self.hist_file = hist_file
221 self.hist_file = hist_file
219
222
220 if self.hist_file == u'':
223 if self.hist_file == u'':
221 # No one has set the hist_file, yet.
224 # No one has set the hist_file, yet.
222 self.hist_file = self._get_hist_file_name(profile)
225 self.hist_file = self._get_hist_file_name(profile)
223
226
224 if sqlite3 is None and self.enabled:
227 if sqlite3 is None and self.enabled:
225 warn("IPython History requires SQLite, your history will not be saved")
228 warn("IPython History requires SQLite, your history will not be saved")
226 self.enabled = False
229 self.enabled = False
227
230
228 self.init_db()
231 self.init_db()
229
232
230 def _get_hist_file_name(self, profile='default'):
233 def _get_hist_file_name(self, profile='default'):
231 """Find the history file for the given profile name.
234 """Find the history file for the given profile name.
232
235
233 This is overridden by the HistoryManager subclass, to use the shell's
236 This is overridden by the HistoryManager subclass, to use the shell's
234 active profile.
237 active profile.
235
238
236 Parameters
239 Parameters
237 ----------
240 ----------
238 profile : str
241 profile : str
239 The name of a profile which has a history file.
242 The name of a profile which has a history file.
240 """
243 """
241 return os.path.join(locate_profile(profile), 'history.sqlite')
244 return os.path.join(locate_profile(profile), 'history.sqlite')
242
245
243 @catch_corrupt_db
246 @catch_corrupt_db
244 def init_db(self):
247 def init_db(self):
245 """Connect to the database, and create tables if necessary."""
248 """Connect to the database, and create tables if necessary."""
246 if not self.enabled:
249 if not self.enabled:
247 self.db = DummyDB()
250 self.db = DummyDB()
248 return
251 return
249
252
250 # use detect_types so that timestamps return datetime objects
253 # use detect_types so that timestamps return datetime objects
251 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
254 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
252 kwargs.update(self.connection_options)
255 kwargs.update(self.connection_options)
253 self.db = sqlite3.connect(self.hist_file, **kwargs)
256 self.db = sqlite3.connect(self.hist_file, **kwargs)
254 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
257 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
255 primary key autoincrement, start timestamp,
258 primary key autoincrement, start timestamp,
256 end timestamp, num_cmds integer, remark text)""")
259 end timestamp, num_cmds integer, remark text)""")
257 self.db.execute("""CREATE TABLE IF NOT EXISTS history
260 self.db.execute("""CREATE TABLE IF NOT EXISTS history
258 (session integer, line integer, source text, source_raw text,
261 (session integer, line integer, source text, source_raw text,
259 PRIMARY KEY (session, line))""")
262 PRIMARY KEY (session, line))""")
260 # Output history is optional, but ensure the table's there so it can be
263 # Output history is optional, but ensure the table's there so it can be
261 # enabled later.
264 # enabled later.
262 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
265 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
263 (session integer, line integer, output text,
266 (session integer, line integer, output text,
264 PRIMARY KEY (session, line))""")
267 PRIMARY KEY (session, line))""")
265 self.db.commit()
268 self.db.commit()
266 # success! reset corrupt db count
269 # success! reset corrupt db count
267 self._corrupt_db_counter = 0
270 self._corrupt_db_counter = 0
268
271
269 def writeout_cache(self):
272 def writeout_cache(self):
270 """Overridden by HistoryManager to dump the cache before certain
273 """Overridden by HistoryManager to dump the cache before certain
271 database lookups."""
274 database lookups."""
272 pass
275 pass
273
276
274 ## -------------------------------
277 ## -------------------------------
275 ## Methods for retrieving history:
278 ## Methods for retrieving history:
276 ## -------------------------------
279 ## -------------------------------
277 def _run_sql(self, sql, params, raw=True, output=False):
280 def _run_sql(self, sql, params, raw=True, output=False):
278 """Prepares and runs an SQL query for the history database.
281 """Prepares and runs an SQL query for the history database.
279
282
280 Parameters
283 Parameters
281 ----------
284 ----------
282 sql : str
285 sql : str
283 Any filtering expressions to go after SELECT ... FROM ...
286 Any filtering expressions to go after SELECT ... FROM ...
284 params : tuple
287 params : tuple
285 Parameters passed to the SQL query (to replace "?")
288 Parameters passed to the SQL query (to replace "?")
286 raw, output : bool
289 raw, output : bool
287 See :meth:`get_range`
290 See :meth:`get_range`
288
291
289 Returns
292 Returns
290 -------
293 -------
291 Tuples as :meth:`get_range`
294 Tuples as :meth:`get_range`
292 """
295 """
293 toget = 'source_raw' if raw else 'source'
296 toget = 'source_raw' if raw else 'source'
294 sqlfrom = "history"
297 sqlfrom = "history"
295 if output:
298 if output:
296 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
299 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
297 toget = "history.%s, output_history.output" % toget
300 toget = "history.%s, output_history.output" % toget
298 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
301 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
299 (toget, sqlfrom) + sql, params)
302 (toget, sqlfrom) + sql, params)
300 if output: # Regroup into 3-tuples, and parse JSON
303 if output: # Regroup into 3-tuples, and parse JSON
301 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
304 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
302 return cur
305 return cur
303
306
304 @needs_sqlite
307 @needs_sqlite
305 @catch_corrupt_db
308 @catch_corrupt_db
306 def get_session_info(self, session):
309 def get_session_info(self, session):
307 """Get info about a session.
310 """Get info about a session.
308
311
309 Parameters
312 Parameters
310 ----------
313 ----------
311
314
312 session : int
315 session : int
313 Session number to retrieve.
316 Session number to retrieve.
314
317
315 Returns
318 Returns
316 -------
319 -------
317
320
318 session_id : int
321 session_id : int
319 Session ID number
322 Session ID number
320 start : datetime
323 start : datetime
321 Timestamp for the start of the session.
324 Timestamp for the start of the session.
322 end : datetime
325 end : datetime
323 Timestamp for the end of the session, or None if IPython crashed.
326 Timestamp for the end of the session, or None if IPython crashed.
324 num_cmds : int
327 num_cmds : int
325 Number of commands run, or None if IPython crashed.
328 Number of commands run, or None if IPython crashed.
326 remark : unicode
329 remark : unicode
327 A manually set description.
330 A manually set description.
328 """
331 """
329 query = "SELECT * from sessions where session == ?"
332 query = "SELECT * from sessions where session == ?"
330 return self.db.execute(query, (session,)).fetchone()
333 return self.db.execute(query, (session,)).fetchone()
331
334
332 @catch_corrupt_db
335 @catch_corrupt_db
333 def get_last_session_id(self):
336 def get_last_session_id(self):
334 """Get the last session ID currently in the database.
337 """Get the last session ID currently in the database.
335
338
336 Within IPython, this should be the same as the value stored in
339 Within IPython, this should be the same as the value stored in
337 :attr:`HistoryManager.session_number`.
340 :attr:`HistoryManager.session_number`.
338 """
341 """
339 for record in self.get_tail(n=1, include_latest=True):
342 for record in self.get_tail(n=1, include_latest=True):
340 return record[0]
343 return record[0]
341
344
342 @catch_corrupt_db
345 @catch_corrupt_db
343 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
346 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
344 """Get the last n lines from the history database.
347 """Get the last n lines from the history database.
345
348
346 Parameters
349 Parameters
347 ----------
350 ----------
348 n : int
351 n : int
349 The number of lines to get
352 The number of lines to get
350 raw, output : bool
353 raw, output : bool
351 See :meth:`get_range`
354 See :meth:`get_range`
352 include_latest : bool
355 include_latest : bool
353 If False (default), n+1 lines are fetched, and the latest one
356 If False (default), n+1 lines are fetched, and the latest one
354 is discarded. This is intended to be used where the function
357 is discarded. This is intended to be used where the function
355 is called by a user command, which it should not return.
358 is called by a user command, which it should not return.
356
359
357 Returns
360 Returns
358 -------
361 -------
359 Tuples as :meth:`get_range`
362 Tuples as :meth:`get_range`
360 """
363 """
361 self.writeout_cache()
364 self.writeout_cache()
362 if not include_latest:
365 if not include_latest:
363 n += 1
366 n += 1
364 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
367 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
365 (n,), raw=raw, output=output)
368 (n,), raw=raw, output=output)
366 if not include_latest:
369 if not include_latest:
367 return reversed(list(cur)[1:])
370 return reversed(list(cur)[1:])
368 return reversed(list(cur))
371 return reversed(list(cur))
369
372
370 @catch_corrupt_db
373 @catch_corrupt_db
371 def search(self, pattern="*", raw=True, search_raw=True,
374 def search(self, pattern="*", raw=True, search_raw=True,
372 output=False, n=None, unique=False):
375 output=False, n=None, unique=False):
373 """Search the database using unix glob-style matching (wildcards
376 """Search the database using unix glob-style matching (wildcards
374 * and ?).
377 * and ?).
375
378
376 Parameters
379 Parameters
377 ----------
380 ----------
378 pattern : str
381 pattern : str
379 The wildcarded pattern to match when searching
382 The wildcarded pattern to match when searching
380 search_raw : bool
383 search_raw : bool
381 If True, search the raw input, otherwise, the parsed input
384 If True, search the raw input, otherwise, the parsed input
382 raw, output : bool
385 raw, output : bool
383 See :meth:`get_range`
386 See :meth:`get_range`
384 n : None or int
387 n : None or int
385 If an integer is given, it defines the limit of
388 If an integer is given, it defines the limit of
386 returned entries.
389 returned entries.
387 unique : bool
390 unique : bool
388 When it is true, return only unique entries.
391 When it is true, return only unique entries.
389
392
390 Returns
393 Returns
391 -------
394 -------
392 Tuples as :meth:`get_range`
395 Tuples as :meth:`get_range`
393 """
396 """
394 tosearch = "source_raw" if search_raw else "source"
397 tosearch = "source_raw" if search_raw else "source"
395 if output:
398 if output:
396 tosearch = "history." + tosearch
399 tosearch = "history." + tosearch
397 self.writeout_cache()
400 self.writeout_cache()
398 sqlform = "WHERE %s GLOB ?" % tosearch
401 sqlform = "WHERE %s GLOB ?" % tosearch
399 params = (pattern,)
402 params = (pattern,)
400 if unique:
403 if unique:
401 sqlform += ' GROUP BY {0}'.format(tosearch)
404 sqlform += ' GROUP BY {0}'.format(tosearch)
402 if n is not None:
405 if n is not None:
403 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
406 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
404 params += (n,)
407 params += (n,)
405 elif unique:
408 elif unique:
406 sqlform += " ORDER BY session, line"
409 sqlform += " ORDER BY session, line"
407 cur = self._run_sql(sqlform, params, raw=raw, output=output)
410 cur = self._run_sql(sqlform, params, raw=raw, output=output)
408 if n is not None:
411 if n is not None:
409 return reversed(list(cur))
412 return reversed(list(cur))
410 return cur
413 return cur
411
414
412 @catch_corrupt_db
415 @catch_corrupt_db
413 def get_range(self, session, start=1, stop=None, raw=True,output=False):
416 def get_range(self, session, start=1, stop=None, raw=True,output=False):
414 """Retrieve input by session.
417 """Retrieve input by session.
415
418
416 Parameters
419 Parameters
417 ----------
420 ----------
418 session : int
421 session : int
419 Session number to retrieve.
422 Session number to retrieve.
420 start : int
423 start : int
421 First line to retrieve.
424 First line to retrieve.
422 stop : int
425 stop : int
423 End of line range (excluded from output itself). If None, retrieve
426 End of line range (excluded from output itself). If None, retrieve
424 to the end of the session.
427 to the end of the session.
425 raw : bool
428 raw : bool
426 If True, return untranslated input
429 If True, return untranslated input
427 output : bool
430 output : bool
428 If True, attempt to include output. This will be 'real' Python
431 If True, attempt to include output. This will be 'real' Python
429 objects for the current session, or text reprs from previous
432 objects for the current session, or text reprs from previous
430 sessions if db_log_output was enabled at the time. Where no output
433 sessions if db_log_output was enabled at the time. Where no output
431 is found, None is used.
434 is found, None is used.
432
435
433 Returns
436 Returns
434 -------
437 -------
435 entries
438 entries
436 An iterator over the desired lines. Each line is a 3-tuple, either
439 An iterator over the desired lines. Each line is a 3-tuple, either
437 (session, line, input) if output is False, or
440 (session, line, input) if output is False, or
438 (session, line, (input, output)) if output is True.
441 (session, line, (input, output)) if output is True.
439 """
442 """
440 if stop:
443 if stop:
441 lineclause = "line >= ? AND line < ?"
444 lineclause = "line >= ? AND line < ?"
442 params = (session, start, stop)
445 params = (session, start, stop)
443 else:
446 else:
444 lineclause = "line>=?"
447 lineclause = "line>=?"
445 params = (session, start)
448 params = (session, start)
446
449
447 return self._run_sql("WHERE session==? AND %s" % lineclause,
450 return self._run_sql("WHERE session==? AND %s" % lineclause,
448 params, raw=raw, output=output)
451 params, raw=raw, output=output)
449
452
450 def get_range_by_str(self, rangestr, raw=True, output=False):
453 def get_range_by_str(self, rangestr, raw=True, output=False):
451 """Get lines of history from a string of ranges, as used by magic
454 """Get lines of history from a string of ranges, as used by magic
452 commands %hist, %save, %macro, etc.
455 commands %hist, %save, %macro, etc.
453
456
454 Parameters
457 Parameters
455 ----------
458 ----------
456 rangestr : str
459 rangestr : str
457 A string specifying ranges, e.g. "5 ~2/1-4". See
460 A string specifying ranges, e.g. "5 ~2/1-4". See
458 :func:`magic_history` for full details.
461 :func:`magic_history` for full details.
459 raw, output : bool
462 raw, output : bool
460 As :meth:`get_range`
463 As :meth:`get_range`
461
464
462 Returns
465 Returns
463 -------
466 -------
464 Tuples as :meth:`get_range`
467 Tuples as :meth:`get_range`
465 """
468 """
466 for sess, s, e in extract_hist_ranges(rangestr):
469 for sess, s, e in extract_hist_ranges(rangestr):
467 for line in self.get_range(sess, s, e, raw=raw, output=output):
470 for line in self.get_range(sess, s, e, raw=raw, output=output):
468 yield line
471 yield line
469
472
470
473
471 class HistoryManager(HistoryAccessor):
474 class HistoryManager(HistoryAccessor):
472 """A class to organize all history-related functionality in one place.
475 """A class to organize all history-related functionality in one place.
473 """
476 """
474 # Public interface
477 # Public interface
475
478
476 # An instance of the IPython shell we are attached to
479 # An instance of the IPython shell we are attached to
477 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
480 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
478 allow_none=True)
481 allow_none=True)
479 # Lists to hold processed and raw history. These start with a blank entry
482 # Lists to hold processed and raw history. These start with a blank entry
480 # so that we can index them starting from 1
483 # so that we can index them starting from 1
481 input_hist_parsed = List([""])
484 input_hist_parsed = List([""])
482 input_hist_raw = List([""])
485 input_hist_raw = List([""])
483 # A list of directories visited during session
486 # A list of directories visited during session
484 dir_hist = List()
487 dir_hist = List()
485 @default('dir_hist')
488 @default('dir_hist')
486 def _dir_hist_default(self):
489 def _dir_hist_default(self):
487 try:
490 try:
488 return [py3compat.getcwd()]
491 return [py3compat.getcwd()]
489 except OSError:
492 except OSError:
490 return []
493 return []
491
494
492 # A dict of output history, keyed with ints from the shell's
495 # A dict of output history, keyed with ints from the shell's
493 # execution count.
496 # execution count.
494 output_hist = Dict()
497 output_hist = Dict()
495 # The text/plain repr of outputs.
498 # The text/plain repr of outputs.
496 output_hist_reprs = Dict()
499 output_hist_reprs = Dict()
497
500
498 # The number of the current session in the history database
501 # The number of the current session in the history database
499 session_number = Integer()
502 session_number = Integer()
500
503
501 db_log_output = Bool(False,
504 db_log_output = Bool(False,
502 help="Should the history database include output? (default: no)"
505 help="Should the history database include output? (default: no)"
503 ).tag(config=True)
506 ).tag(config=True)
504 db_cache_size = Integer(0,
507 db_cache_size = Integer(0,
505 help="Write to database every x commands (higher values save disk access & power).\n"
508 help="Write to database every x commands (higher values save disk access & power).\n"
506 "Values of 1 or less effectively disable caching."
509 "Values of 1 or less effectively disable caching."
507 ).tag(config=True)
510 ).tag(config=True)
508 # The input and output caches
511 # The input and output caches
509 db_input_cache = List()
512 db_input_cache = List()
510 db_output_cache = List()
513 db_output_cache = List()
511
514
512 # History saving in separate thread
515 # History saving in separate thread
513 save_thread = Instance('IPython.core.history.HistorySavingThread',
516 save_thread = Instance('IPython.core.history.HistorySavingThread',
514 allow_none=True)
517 allow_none=True)
515 try: # Event is a function returning an instance of _Event...
518 try: # Event is a function returning an instance of _Event...
516 save_flag = Instance(threading._Event, allow_none=True)
519 save_flag = Instance(threading._Event, allow_none=True)
517 except AttributeError: # ...until Python 3.3, when it's a class.
520 except AttributeError: # ...until Python 3.3, when it's a class.
518 save_flag = Instance(threading.Event, allow_none=True)
521 save_flag = Instance(threading.Event, allow_none=True)
519
522
520 # Private interface
523 # Private interface
521 # Variables used to store the three last inputs from the user. On each new
524 # Variables used to store the three last inputs from the user. On each new
522 # history update, we populate the user's namespace with these, shifted as
525 # history update, we populate the user's namespace with these, shifted as
523 # necessary.
526 # necessary.
524 _i00 = Unicode(u'')
527 _i00 = Unicode(u'')
525 _i = Unicode(u'')
528 _i = Unicode(u'')
526 _ii = Unicode(u'')
529 _ii = Unicode(u'')
527 _iii = Unicode(u'')
530 _iii = Unicode(u'')
528
531
529 # A regex matching all forms of the exit command, so that we don't store
532 # A regex matching all forms of the exit command, so that we don't store
530 # them in the history (it's annoying to rewind the first entry and land on
533 # them in the history (it's annoying to rewind the first entry and land on
531 # an exit call).
534 # an exit call).
532 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
535 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
533
536
534 def __init__(self, shell=None, config=None, **traits):
537 def __init__(self, shell=None, config=None, **traits):
535 """Create a new history manager associated with a shell instance.
538 """Create a new history manager associated with a shell instance.
536 """
539 """
537 # We need a pointer back to the shell for various tasks.
540 # We need a pointer back to the shell for various tasks.
538 super(HistoryManager, self).__init__(shell=shell, config=config,
541 super(HistoryManager, self).__init__(shell=shell, config=config,
539 **traits)
542 **traits)
540 self.save_flag = threading.Event()
543 self.save_flag = threading.Event()
541 self.db_input_cache_lock = threading.Lock()
544 self.db_input_cache_lock = threading.Lock()
542 self.db_output_cache_lock = threading.Lock()
545 self.db_output_cache_lock = threading.Lock()
543
546
544 try:
547 try:
545 self.new_session()
548 self.new_session()
546 except OperationalError:
549 except OperationalError:
547 self.log.error("Failed to create history session in %s. History will not be saved.",
550 self.log.error("Failed to create history session in %s. History will not be saved.",
548 self.hist_file, exc_info=True)
551 self.hist_file, exc_info=True)
549 self.hist_file = ':memory:'
552 self.hist_file = ':memory:'
550
553
551 if self.enabled and self.hist_file != ':memory:':
554 if self.enabled and self.hist_file != ':memory:':
552 self.save_thread = HistorySavingThread(self)
555 self.save_thread = HistorySavingThread(self)
553 self.save_thread.start()
556 self.save_thread.start()
554
557
555 def _get_hist_file_name(self, profile=None):
558 def _get_hist_file_name(self, profile=None):
556 """Get default history file name based on the Shell's profile.
559 """Get default history file name based on the Shell's profile.
557
560
558 The profile parameter is ignored, but must exist for compatibility with
561 The profile parameter is ignored, but must exist for compatibility with
559 the parent class."""
562 the parent class."""
560 profile_dir = self.shell.profile_dir.location
563 profile_dir = self.shell.profile_dir.location
561 return os.path.join(profile_dir, 'history.sqlite')
564 return os.path.join(profile_dir, 'history.sqlite')
562
565
563 @needs_sqlite
566 @needs_sqlite
564 def new_session(self, conn=None):
567 def new_session(self, conn=None):
565 """Get a new session number."""
568 """Get a new session number."""
566 if conn is None:
569 if conn is None:
567 conn = self.db
570 conn = self.db
568
571
569 with conn:
572 with conn:
570 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
573 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
571 NULL, "") """, (datetime.datetime.now(),))
574 NULL, "") """, (datetime.datetime.now(),))
572 self.session_number = cur.lastrowid
575 self.session_number = cur.lastrowid
573
576
574 def end_session(self):
577 def end_session(self):
575 """Close the database session, filling in the end time and line count."""
578 """Close the database session, filling in the end time and line count."""
576 self.writeout_cache()
579 self.writeout_cache()
577 with self.db:
580 with self.db:
578 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
581 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
579 session==?""", (datetime.datetime.now(),
582 session==?""", (datetime.datetime.now(),
580 len(self.input_hist_parsed)-1, self.session_number))
583 len(self.input_hist_parsed)-1, self.session_number))
581 self.session_number = 0
584 self.session_number = 0
582
585
583 def name_session(self, name):
586 def name_session(self, name):
584 """Give the current session a name in the history database."""
587 """Give the current session a name in the history database."""
585 with self.db:
588 with self.db:
586 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
589 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
587 (name, self.session_number))
590 (name, self.session_number))
588
591
589 def reset(self, new_session=True):
592 def reset(self, new_session=True):
590 """Clear the session history, releasing all object references, and
593 """Clear the session history, releasing all object references, and
591 optionally open a new session."""
594 optionally open a new session."""
592 self.output_hist.clear()
595 self.output_hist.clear()
593 # The directory history can't be completely empty
596 # The directory history can't be completely empty
594 self.dir_hist[:] = [py3compat.getcwd()]
597 self.dir_hist[:] = [py3compat.getcwd()]
595
598
596 if new_session:
599 if new_session:
597 if self.session_number:
600 if self.session_number:
598 self.end_session()
601 self.end_session()
599 self.input_hist_parsed[:] = [""]
602 self.input_hist_parsed[:] = [""]
600 self.input_hist_raw[:] = [""]
603 self.input_hist_raw[:] = [""]
601 self.new_session()
604 self.new_session()
602
605
603 # ------------------------------
606 # ------------------------------
604 # Methods for retrieving history
607 # Methods for retrieving history
605 # ------------------------------
608 # ------------------------------
606 def get_session_info(self, session=0):
609 def get_session_info(self, session=0):
607 """Get info about a session.
610 """Get info about a session.
608
611
609 Parameters
612 Parameters
610 ----------
613 ----------
611
614
612 session : int
615 session : int
613 Session number to retrieve. The current session is 0, and negative
616 Session number to retrieve. The current session is 0, and negative
614 numbers count back from current session, so -1 is the previous session.
617 numbers count back from current session, so -1 is the previous session.
615
618
616 Returns
619 Returns
617 -------
620 -------
618
621
619 session_id : int
622 session_id : int
620 Session ID number
623 Session ID number
621 start : datetime
624 start : datetime
622 Timestamp for the start of the session.
625 Timestamp for the start of the session.
623 end : datetime
626 end : datetime
624 Timestamp for the end of the session, or None if IPython crashed.
627 Timestamp for the end of the session, or None if IPython crashed.
625 num_cmds : int
628 num_cmds : int
626 Number of commands run, or None if IPython crashed.
629 Number of commands run, or None if IPython crashed.
627 remark : unicode
630 remark : unicode
628 A manually set description.
631 A manually set description.
629 """
632 """
630 if session <= 0:
633 if session <= 0:
631 session += self.session_number
634 session += self.session_number
632
635
633 return super(HistoryManager, self).get_session_info(session=session)
636 return super(HistoryManager, self).get_session_info(session=session)
634
637
635 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
638 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
636 """Get input and output history from the current session. Called by
639 """Get input and output history from the current session. Called by
637 get_range, and takes similar parameters."""
640 get_range, and takes similar parameters."""
638 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
641 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
639
642
640 n = len(input_hist)
643 n = len(input_hist)
641 if start < 0:
644 if start < 0:
642 start += n
645 start += n
643 if not stop or (stop > n):
646 if not stop or (stop > n):
644 stop = n
647 stop = n
645 elif stop < 0:
648 elif stop < 0:
646 stop += n
649 stop += n
647
650
648 for i in range(start, stop):
651 for i in range(start, stop):
649 if output:
652 if output:
650 line = (input_hist[i], self.output_hist_reprs.get(i))
653 line = (input_hist[i], self.output_hist_reprs.get(i))
651 else:
654 else:
652 line = input_hist[i]
655 line = input_hist[i]
653 yield (0, i, line)
656 yield (0, i, line)
654
657
655 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
658 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
656 """Retrieve input by session.
659 """Retrieve input by session.
657
660
658 Parameters
661 Parameters
659 ----------
662 ----------
660 session : int
663 session : int
661 Session number to retrieve. The current session is 0, and negative
664 Session number to retrieve. The current session is 0, and negative
662 numbers count back from current session, so -1 is previous session.
665 numbers count back from current session, so -1 is previous session.
663 start : int
666 start : int
664 First line to retrieve.
667 First line to retrieve.
665 stop : int
668 stop : int
666 End of line range (excluded from output itself). If None, retrieve
669 End of line range (excluded from output itself). If None, retrieve
667 to the end of the session.
670 to the end of the session.
668 raw : bool
671 raw : bool
669 If True, return untranslated input
672 If True, return untranslated input
670 output : bool
673 output : bool
671 If True, attempt to include output. This will be 'real' Python
674 If True, attempt to include output. This will be 'real' Python
672 objects for the current session, or text reprs from previous
675 objects for the current session, or text reprs from previous
673 sessions if db_log_output was enabled at the time. Where no output
676 sessions if db_log_output was enabled at the time. Where no output
674 is found, None is used.
677 is found, None is used.
675
678
676 Returns
679 Returns
677 -------
680 -------
678 entries
681 entries
679 An iterator over the desired lines. Each line is a 3-tuple, either
682 An iterator over the desired lines. Each line is a 3-tuple, either
680 (session, line, input) if output is False, or
683 (session, line, input) if output is False, or
681 (session, line, (input, output)) if output is True.
684 (session, line, (input, output)) if output is True.
682 """
685 """
683 if session <= 0:
686 if session <= 0:
684 session += self.session_number
687 session += self.session_number
685 if session==self.session_number: # Current session
688 if session==self.session_number: # Current session
686 return self._get_range_session(start, stop, raw, output)
689 return self._get_range_session(start, stop, raw, output)
687 return super(HistoryManager, self).get_range(session, start, stop, raw,
690 return super(HistoryManager, self).get_range(session, start, stop, raw,
688 output)
691 output)
689
692
690 ## ----------------------------
693 ## ----------------------------
691 ## Methods for storing history:
694 ## Methods for storing history:
692 ## ----------------------------
695 ## ----------------------------
693 def store_inputs(self, line_num, source, source_raw=None):
696 def store_inputs(self, line_num, source, source_raw=None):
694 """Store source and raw input in history and create input cache
697 """Store source and raw input in history and create input cache
695 variables ``_i*``.
698 variables ``_i*``.
696
699
697 Parameters
700 Parameters
698 ----------
701 ----------
699 line_num : int
702 line_num : int
700 The prompt number of this input.
703 The prompt number of this input.
701
704
702 source : str
705 source : str
703 Python input.
706 Python input.
704
707
705 source_raw : str, optional
708 source_raw : str, optional
706 If given, this is the raw input without any IPython transformations
709 If given, this is the raw input without any IPython transformations
707 applied to it. If not given, ``source`` is used.
710 applied to it. If not given, ``source`` is used.
708 """
711 """
709 if source_raw is None:
712 if source_raw is None:
710 source_raw = source
713 source_raw = source
711 source = source.rstrip('\n')
714 source = source.rstrip('\n')
712 source_raw = source_raw.rstrip('\n')
715 source_raw = source_raw.rstrip('\n')
713
716
714 # do not store exit/quit commands
717 # do not store exit/quit commands
715 if self._exit_re.match(source_raw.strip()):
718 if self._exit_re.match(source_raw.strip()):
716 return
719 return
717
720
718 self.input_hist_parsed.append(source)
721 self.input_hist_parsed.append(source)
719 self.input_hist_raw.append(source_raw)
722 self.input_hist_raw.append(source_raw)
720
723
721 with self.db_input_cache_lock:
724 with self.db_input_cache_lock:
722 self.db_input_cache.append((line_num, source, source_raw))
725 self.db_input_cache.append((line_num, source, source_raw))
723 # Trigger to flush cache and write to DB.
726 # Trigger to flush cache and write to DB.
724 if len(self.db_input_cache) >= self.db_cache_size:
727 if len(self.db_input_cache) >= self.db_cache_size:
725 self.save_flag.set()
728 self.save_flag.set()
726
729
727 # update the auto _i variables
730 # update the auto _i variables
728 self._iii = self._ii
731 self._iii = self._ii
729 self._ii = self._i
732 self._ii = self._i
730 self._i = self._i00
733 self._i = self._i00
731 self._i00 = source_raw
734 self._i00 = source_raw
732
735
733 # hackish access to user namespace to create _i1,_i2... dynamically
736 # hackish access to user namespace to create _i1,_i2... dynamically
734 new_i = '_i%s' % line_num
737 new_i = '_i%s' % line_num
735 to_main = {'_i': self._i,
738 to_main = {'_i': self._i,
736 '_ii': self._ii,
739 '_ii': self._ii,
737 '_iii': self._iii,
740 '_iii': self._iii,
738 new_i : self._i00 }
741 new_i : self._i00 }
739
742
740 if self.shell is not None:
743 if self.shell is not None:
741 self.shell.push(to_main, interactive=False)
744 self.shell.push(to_main, interactive=False)
742
745
743 def store_output(self, line_num):
746 def store_output(self, line_num):
744 """If database output logging is enabled, this saves all the
747 """If database output logging is enabled, this saves all the
745 outputs from the indicated prompt number to the database. It's
748 outputs from the indicated prompt number to the database. It's
746 called by run_cell after code has been executed.
749 called by run_cell after code has been executed.
747
750
748 Parameters
751 Parameters
749 ----------
752 ----------
750 line_num : int
753 line_num : int
751 The line number from which to save outputs
754 The line number from which to save outputs
752 """
755 """
753 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
756 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
754 return
757 return
755 output = self.output_hist_reprs[line_num]
758 output = self.output_hist_reprs[line_num]
756
759
757 with self.db_output_cache_lock:
760 with self.db_output_cache_lock:
758 self.db_output_cache.append((line_num, output))
761 self.db_output_cache.append((line_num, output))
759 if self.db_cache_size <= 1:
762 if self.db_cache_size <= 1:
760 self.save_flag.set()
763 self.save_flag.set()
761
764
762 def _writeout_input_cache(self, conn):
765 def _writeout_input_cache(self, conn):
763 with conn:
766 with conn:
764 for line in self.db_input_cache:
767 for line in self.db_input_cache:
765 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
768 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
766 (self.session_number,)+line)
769 (self.session_number,)+line)
767
770
768 def _writeout_output_cache(self, conn):
771 def _writeout_output_cache(self, conn):
769 with conn:
772 with conn:
770 for line in self.db_output_cache:
773 for line in self.db_output_cache:
771 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
774 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
772 (self.session_number,)+line)
775 (self.session_number,)+line)
773
776
774 @needs_sqlite
777 @needs_sqlite
775 def writeout_cache(self, conn=None):
778 def writeout_cache(self, conn=None):
776 """Write any entries in the cache to the database."""
779 """Write any entries in the cache to the database."""
777 if conn is None:
780 if conn is None:
778 conn = self.db
781 conn = self.db
779
782
780 with self.db_input_cache_lock:
783 with self.db_input_cache_lock:
781 try:
784 try:
782 self._writeout_input_cache(conn)
785 self._writeout_input_cache(conn)
783 except sqlite3.IntegrityError:
786 except sqlite3.IntegrityError:
784 self.new_session(conn)
787 self.new_session(conn)
785 print("ERROR! Session/line number was not unique in",
788 print("ERROR! Session/line number was not unique in",
786 "database. History logging moved to new session",
789 "database. History logging moved to new session",
787 self.session_number)
790 self.session_number)
788 try:
791 try:
789 # Try writing to the new session. If this fails, don't
792 # Try writing to the new session. If this fails, don't
790 # recurse
793 # recurse
791 self._writeout_input_cache(conn)
794 self._writeout_input_cache(conn)
792 except sqlite3.IntegrityError:
795 except sqlite3.IntegrityError:
793 pass
796 pass
794 finally:
797 finally:
795 self.db_input_cache = []
798 self.db_input_cache = []
796
799
797 with self.db_output_cache_lock:
800 with self.db_output_cache_lock:
798 try:
801 try:
799 self._writeout_output_cache(conn)
802 self._writeout_output_cache(conn)
800 except sqlite3.IntegrityError:
803 except sqlite3.IntegrityError:
801 print("!! Session/line number for output was not unique",
804 print("!! Session/line number for output was not unique",
802 "in database. Output will not be stored.")
805 "in database. Output will not be stored.")
803 finally:
806 finally:
804 self.db_output_cache = []
807 self.db_output_cache = []
805
808
806
809
807 class HistorySavingThread(threading.Thread):
810 class HistorySavingThread(threading.Thread):
808 """This thread takes care of writing history to the database, so that
811 """This thread takes care of writing history to the database, so that
809 the UI isn't held up while that happens.
812 the UI isn't held up while that happens.
810
813
811 It waits for the HistoryManager's save_flag to be set, then writes out
814 It waits for the HistoryManager's save_flag to be set, then writes out
812 the history cache. The main thread is responsible for setting the flag when
815 the history cache. The main thread is responsible for setting the flag when
813 the cache size reaches a defined threshold."""
816 the cache size reaches a defined threshold."""
814 daemon = True
817 daemon = True
815 stop_now = False
818 stop_now = False
816 enabled = True
819 enabled = True
817 def __init__(self, history_manager):
820 def __init__(self, history_manager):
818 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
821 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
819 self.history_manager = history_manager
822 self.history_manager = history_manager
820 self.enabled = history_manager.enabled
823 self.enabled = history_manager.enabled
821 atexit.register(self.stop)
824 atexit.register(self.stop)
822
825
823 @needs_sqlite
826 @needs_sqlite
824 def run(self):
827 def run(self):
825 # We need a separate db connection per thread:
828 # We need a separate db connection per thread:
826 try:
829 try:
827 self.db = sqlite3.connect(self.history_manager.hist_file,
830 self.db = sqlite3.connect(self.history_manager.hist_file,
828 **self.history_manager.connection_options
831 **self.history_manager.connection_options
829 )
832 )
830 while True:
833 while True:
831 self.history_manager.save_flag.wait()
834 self.history_manager.save_flag.wait()
832 if self.stop_now:
835 if self.stop_now:
833 self.db.close()
836 self.db.close()
834 return
837 return
835 self.history_manager.save_flag.clear()
838 self.history_manager.save_flag.clear()
836 self.history_manager.writeout_cache(self.db)
839 self.history_manager.writeout_cache(self.db)
837 except Exception as e:
840 except Exception as e:
838 print(("The history saving thread hit an unexpected error (%s)."
841 print(("The history saving thread hit an unexpected error (%s)."
839 "History will not be written to the database.") % repr(e))
842 "History will not be written to the database.") % repr(e))
840
843
841 def stop(self):
844 def stop(self):
842 """This can be called from the main thread to safely stop this thread.
845 """This can be called from the main thread to safely stop this thread.
843
846
844 Note that it does not attempt to write out remaining history before
847 Note that it does not attempt to write out remaining history before
845 exiting. That should be done by calling the HistoryManager's
848 exiting. That should be done by calling the HistoryManager's
846 end_session method."""
849 end_session method."""
847 self.stop_now = True
850 self.stop_now = True
848 self.history_manager.save_flag.set()
851 self.history_manager.save_flag.set()
849 self.join()
852 self.join()
850
853
851
854
852 # To match, e.g. ~5/8-~2/3
855 # To match, e.g. ~5/8-~2/3
853 range_re = re.compile(r"""
856 range_re = re.compile(r"""
854 ((?P<startsess>~?\d+)/)?
857 ((?P<startsess>~?\d+)/)?
855 (?P<start>\d+)?
858 (?P<start>\d+)?
856 ((?P<sep>[\-:])
859 ((?P<sep>[\-:])
857 ((?P<endsess>~?\d+)/)?
860 ((?P<endsess>~?\d+)/)?
858 (?P<end>\d+))?
861 (?P<end>\d+))?
859 $""", re.VERBOSE)
862 $""", re.VERBOSE)
860
863
861
864
862 def extract_hist_ranges(ranges_str):
865 def extract_hist_ranges(ranges_str):
863 """Turn a string of history ranges into 3-tuples of (session, start, stop).
866 """Turn a string of history ranges into 3-tuples of (session, start, stop).
864
867
865 Examples
868 Examples
866 --------
869 --------
867 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
870 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
868 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
871 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
869 """
872 """
870 for range_str in ranges_str.split():
873 for range_str in ranges_str.split():
871 rmatch = range_re.match(range_str)
874 rmatch = range_re.match(range_str)
872 if not rmatch:
875 if not rmatch:
873 continue
876 continue
874 start = rmatch.group("start")
877 start = rmatch.group("start")
875 if start:
878 if start:
876 start = int(start)
879 start = int(start)
877 end = rmatch.group("end")
880 end = rmatch.group("end")
878 # If no end specified, get (a, a + 1)
881 # If no end specified, get (a, a + 1)
879 end = int(end) if end else start + 1
882 end = int(end) if end else start + 1
880 else: # start not specified
883 else: # start not specified
881 if not rmatch.group('startsess'): # no startsess
884 if not rmatch.group('startsess'): # no startsess
882 continue
885 continue
883 start = 1
886 start = 1
884 end = None # provide the entire session hist
887 end = None # provide the entire session hist
885
888
886 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
889 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
887 end += 1
890 end += 1
888 startsess = rmatch.group("startsess") or "0"
891 startsess = rmatch.group("startsess") or "0"
889 endsess = rmatch.group("endsess") or startsess
892 endsess = rmatch.group("endsess") or startsess
890 startsess = int(startsess.replace("~","-"))
893 startsess = int(startsess.replace("~","-"))
891 endsess = int(endsess.replace("~","-"))
894 endsess = int(endsess.replace("~","-"))
892 assert endsess >= startsess, "start session must be earlier than end session"
895 assert endsess >= startsess, "start session must be earlier than end session"
893
896
894 if endsess == startsess:
897 if endsess == startsess:
895 yield (startsess, start, end)
898 yield (startsess, start, end)
896 continue
899 continue
897 # Multiple sessions in one range:
900 # Multiple sessions in one range:
898 yield (startsess, start, None)
901 yield (startsess, start, None)
899 for sess in range(startsess+1, endsess):
902 for sess in range(startsess+1, endsess):
900 yield (sess, 1, None)
903 yield (sess, 1, None)
901 yield (endsess, 1, end)
904 yield (endsess, 1, end)
902
905
903
906
904 def _format_lineno(session, line):
907 def _format_lineno(session, line):
905 """Helper function to format line numbers properly."""
908 """Helper function to format line numbers properly."""
906 if session == 0:
909 if session == 0:
907 return str(line)
910 return str(line)
908 return "%s#%s" % (session, line)
911 return "%s#%s" % (session, line)
General Comments 0
You need to be logged in to leave comments. Login now