##// END OF EJS Templates
Merge pull request #9418 from minrk/catch-bad-db...
Matthias Bussonnier -
r22274:3a1f586b merge
parent child Browse files
Show More
@@ -1,873 +1,904 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 from __future__ import print_function
6 from __future__ import print_function
7
7
8 import atexit
8 import atexit
9 import datetime
9 import datetime
10 import os
10 import os
11 import re
11 import re
12 try:
12 try:
13 import sqlite3
13 import sqlite3
14 except ImportError:
14 except ImportError:
15 try:
15 try:
16 from pysqlite2 import dbapi2 as sqlite3
16 from pysqlite2 import dbapi2 as sqlite3
17 except ImportError:
17 except ImportError:
18 sqlite3 = None
18 sqlite3 = None
19 import sys
20 import threading
19 import threading
21
20
22 from traitlets.config.configurable import LoggingConfigurable
21 from traitlets.config.configurable import LoggingConfigurable
23 from decorator import decorator
22 from decorator import decorator
24 from IPython.utils.decorators import undoc
23 from IPython.utils.decorators import undoc
25 from IPython.utils.path import locate_profile
24 from IPython.utils.path import locate_profile
26 from IPython.utils import py3compat
25 from IPython.utils import py3compat
27 from traitlets import (
26 from traitlets import (
28 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
27 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
29 )
28 )
30 from warnings import warn
29 from warnings import warn
31
30
32 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
33 # Classes and functions
32 # Classes and functions
34 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
35
34
36 @undoc
35 @undoc
37 class DummyDB(object):
36 class DummyDB(object):
38 """Dummy DB that will act as a black hole for history.
37 """Dummy DB that will act as a black hole for history.
39
38
40 Only used in the absence of sqlite"""
39 Only used in the absence of sqlite"""
41 def execute(*args, **kwargs):
40 def execute(*args, **kwargs):
42 return []
41 return []
43
42
44 def commit(self, *args, **kwargs):
43 def commit(self, *args, **kwargs):
45 pass
44 pass
46
45
47 def __enter__(self, *args, **kwargs):
46 def __enter__(self, *args, **kwargs):
48 pass
47 pass
49
48
50 def __exit__(self, *args, **kwargs):
49 def __exit__(self, *args, **kwargs):
51 pass
50 pass
52
51
53
52
54 @decorator
53 @decorator
55 def needs_sqlite(f, self, *a, **kw):
54 def needs_sqlite(f, self, *a, **kw):
56 """Decorator: return an empty list in the absence of sqlite."""
55 """Decorator: return an empty list in the absence of sqlite."""
57 if sqlite3 is None or not self.enabled:
56 if sqlite3 is None or not self.enabled:
58 return []
57 return []
59 else:
58 else:
60 return f(self, *a, **kw)
59 return f(self, *a, **kw)
61
60
62
61
63 if sqlite3 is not None:
62 if sqlite3 is not None:
64 DatabaseError = sqlite3.DatabaseError
63 DatabaseError = sqlite3.DatabaseError
65 OperationalError = sqlite3.OperationalError
64 OperationalError = sqlite3.OperationalError
66 else:
65 else:
67 @undoc
66 @undoc
68 class DatabaseError(Exception):
67 class DatabaseError(Exception):
69 "Dummy exception when sqlite could not be imported. Should never occur."
68 "Dummy exception when sqlite could not be imported. Should never occur."
70
69
71 @undoc
70 @undoc
72 class OperationalError(Exception):
71 class OperationalError(Exception):
73 "Dummy exception when sqlite could not be imported. Should never occur."
72 "Dummy exception when sqlite could not be imported. Should never occur."
74
73
74 # use 16kB as threshold for whether a corrupt history db should be saved
75 # that should be at least 100 entries or so
76 _SAVE_DB_SIZE = 16384
77
75 @decorator
78 @decorator
76 def catch_corrupt_db(f, self, *a, **kw):
79 def catch_corrupt_db(f, self, *a, **kw):
77 """A decorator which wraps HistoryAccessor method calls to catch errors from
80 """A decorator which wraps HistoryAccessor method calls to catch errors from
78 a corrupt SQLite database, move the old database out of the way, and create
81 a corrupt SQLite database, move the old database out of the way, and create
79 a new one.
82 a new one.
83
84 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
85 not just a corrupt file.
80 """
86 """
81 try:
87 try:
82 return f(self, *a, **kw)
88 return f(self, *a, **kw)
83 except (DatabaseError, OperationalError) as e:
89 except (DatabaseError, OperationalError) as e:
84 if os.path.isfile(self.hist_file):
90 self._corrupt_db_counter += 1
85 # Try to move the file out of the way
91 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
86 base,ext = os.path.splitext(self.hist_file)
92 if self.hist_file != ':memory:':
87 newpath = base + '-corrupt' + ext
93 if self._corrupt_db_counter > self._corrupt_db_limit:
88 os.rename(self.hist_file, newpath)
94 self.hist_file = ':memory:'
89 print("ERROR! History file wasn't a valid SQLite database (%s)." % e,
95 self.log.error("Failed to load history too many times, history will not be saved.")
90 "It was moved to %s" % newpath, "and a new file created.", file=sys.stderr)
96 elif os.path.isfile(self.hist_file):
97 # move the file out of the way
98 base, ext = os.path.splitext(self.hist_file)
99 size = os.stat(self.hist_file).st_size
100 if size >= _SAVE_DB_SIZE:
101 # if there's significant content, avoid clobbering
102 now = datetime.datetime.now().isoformat().replace(':', '.')
103 newpath = base + '-corrupt-' + now + ext
104 # don't clobber previous corrupt backups
105 for i in range(100):
106 if not os.path.isfile(newpath):
107 break
108 else:
109 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
110 else:
111 # not much content, possibly empty; don't worry about clobbering
112 # maybe we should just delete it?
113 newpath = base + '-corrupt' + ext
114 os.rename(self.hist_file, newpath)
115 self.log.error("History file was moved to %s and a new file created.", newpath)
91 self.init_db()
116 self.init_db()
92 return []
117 return []
93
94 else:
118 else:
95 # The hist_file is probably :memory: or something else.
119 # Failed with :memory:, something serious is wrong
96 raise
120 raise
97
121
98 class HistoryAccessorBase(LoggingConfigurable):
122 class HistoryAccessorBase(LoggingConfigurable):
99 """An abstract class for History Accessors """
123 """An abstract class for History Accessors """
100
124
101 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
125 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
102 raise NotImplementedError
126 raise NotImplementedError
103
127
104 def search(self, pattern="*", raw=True, search_raw=True,
128 def search(self, pattern="*", raw=True, search_raw=True,
105 output=False, n=None, unique=False):
129 output=False, n=None, unique=False):
106 raise NotImplementedError
130 raise NotImplementedError
107
131
108 def get_range(self, session, start=1, stop=None, raw=True,output=False):
132 def get_range(self, session, start=1, stop=None, raw=True,output=False):
109 raise NotImplementedError
133 raise NotImplementedError
110
134
111 def get_range_by_str(self, rangestr, raw=True, output=False):
135 def get_range_by_str(self, rangestr, raw=True, output=False):
112 raise NotImplementedError
136 raise NotImplementedError
113
137
114
138
115 class HistoryAccessor(HistoryAccessorBase):
139 class HistoryAccessor(HistoryAccessorBase):
116 """Access the history database without adding to it.
140 """Access the history database without adding to it.
117
141
118 This is intended for use by standalone history tools. IPython shells use
142 This is intended for use by standalone history tools. IPython shells use
119 HistoryManager, below, which is a subclass of this."""
143 HistoryManager, below, which is a subclass of this."""
120
144
145 # counter for init_db retries, so we don't keep trying over and over
146 _corrupt_db_counter = 0
147 # after two failures, fallback on :memory:
148 _corrupt_db_limit = 2
149
121 # String holding the path to the history file
150 # String holding the path to the history file
122 hist_file = Unicode(config=True,
151 hist_file = Unicode(config=True,
123 help="""Path to file to use for SQLite history database.
152 help="""Path to file to use for SQLite history database.
124
153
125 By default, IPython will put the history database in the IPython
154 By default, IPython will put the history database in the IPython
126 profile directory. If you would rather share one history among
155 profile directory. If you would rather share one history among
127 profiles, you can set this value in each, so that they are consistent.
156 profiles, you can set this value in each, so that they are consistent.
128
157
129 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
158 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
130 mounts. If you see IPython hanging, try setting this to something on a
159 mounts. If you see IPython hanging, try setting this to something on a
131 local disk, e.g::
160 local disk, e.g::
132
161
133 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
162 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
134
163
135 """)
164 """)
136
165
137 enabled = Bool(True, config=True,
166 enabled = Bool(True, config=True,
138 help="""enable the SQLite history
167 help="""enable the SQLite history
139
168
140 set enabled=False to disable the SQLite history,
169 set enabled=False to disable the SQLite history,
141 in which case there will be no stored history, no SQLite connection,
170 in which case there will be no stored history, no SQLite connection,
142 and no background saving thread. This may be necessary in some
171 and no background saving thread. This may be necessary in some
143 threaded environments where IPython is embedded.
172 threaded environments where IPython is embedded.
144 """
173 """
145 )
174 )
146
175
147 connection_options = Dict(config=True,
176 connection_options = Dict(config=True,
148 help="""Options for configuring the SQLite connection
177 help="""Options for configuring the SQLite connection
149
178
150 These options are passed as keyword args to sqlite3.connect
179 These options are passed as keyword args to sqlite3.connect
151 when establishing database conenctions.
180 when establishing database conenctions.
152 """
181 """
153 )
182 )
154
183
155 # The SQLite database
184 # The SQLite database
156 db = Any()
185 db = Any()
157 def _db_changed(self, name, old, new):
186 def _db_changed(self, name, old, new):
158 """validate the db, since it can be an Instance of two different types"""
187 """validate the db, since it can be an Instance of two different types"""
159 connection_types = (DummyDB,)
188 connection_types = (DummyDB,)
160 if sqlite3 is not None:
189 if sqlite3 is not None:
161 connection_types = (DummyDB, sqlite3.Connection)
190 connection_types = (DummyDB, sqlite3.Connection)
162 if not isinstance(new, connection_types):
191 if not isinstance(new, connection_types):
163 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
192 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
164 (self.__class__.__name__, new)
193 (self.__class__.__name__, new)
165 raise TraitError(msg)
194 raise TraitError(msg)
166
195
167 def __init__(self, profile='default', hist_file=u'', **traits):
196 def __init__(self, profile='default', hist_file=u'', **traits):
168 """Create a new history accessor.
197 """Create a new history accessor.
169
198
170 Parameters
199 Parameters
171 ----------
200 ----------
172 profile : str
201 profile : str
173 The name of the profile from which to open history.
202 The name of the profile from which to open history.
174 hist_file : str
203 hist_file : str
175 Path to an SQLite history database stored by IPython. If specified,
204 Path to an SQLite history database stored by IPython. If specified,
176 hist_file overrides profile.
205 hist_file overrides profile.
177 config : :class:`~traitlets.config.loader.Config`
206 config : :class:`~traitlets.config.loader.Config`
178 Config object. hist_file can also be set through this.
207 Config object. hist_file can also be set through this.
179 """
208 """
180 # We need a pointer back to the shell for various tasks.
209 # We need a pointer back to the shell for various tasks.
181 super(HistoryAccessor, self).__init__(**traits)
210 super(HistoryAccessor, self).__init__(**traits)
182 # defer setting hist_file from kwarg until after init,
211 # defer setting hist_file from kwarg until after init,
183 # otherwise the default kwarg value would clobber any value
212 # otherwise the default kwarg value would clobber any value
184 # set by config
213 # set by config
185 if hist_file:
214 if hist_file:
186 self.hist_file = hist_file
215 self.hist_file = hist_file
187
216
188 if self.hist_file == u'':
217 if self.hist_file == u'':
189 # No one has set the hist_file, yet.
218 # No one has set the hist_file, yet.
190 self.hist_file = self._get_hist_file_name(profile)
219 self.hist_file = self._get_hist_file_name(profile)
191
220
192 if sqlite3 is None and self.enabled:
221 if sqlite3 is None and self.enabled:
193 warn("IPython History requires SQLite, your history will not be saved")
222 warn("IPython History requires SQLite, your history will not be saved")
194 self.enabled = False
223 self.enabled = False
195
224
196 self.init_db()
225 self.init_db()
197
226
198 def _get_hist_file_name(self, profile='default'):
227 def _get_hist_file_name(self, profile='default'):
199 """Find the history file for the given profile name.
228 """Find the history file for the given profile name.
200
229
201 This is overridden by the HistoryManager subclass, to use the shell's
230 This is overridden by the HistoryManager subclass, to use the shell's
202 active profile.
231 active profile.
203
232
204 Parameters
233 Parameters
205 ----------
234 ----------
206 profile : str
235 profile : str
207 The name of a profile which has a history file.
236 The name of a profile which has a history file.
208 """
237 """
209 return os.path.join(locate_profile(profile), 'history.sqlite')
238 return os.path.join(locate_profile(profile), 'history.sqlite')
210
239
211 @catch_corrupt_db
240 @catch_corrupt_db
212 def init_db(self):
241 def init_db(self):
213 """Connect to the database, and create tables if necessary."""
242 """Connect to the database, and create tables if necessary."""
214 if not self.enabled:
243 if not self.enabled:
215 self.db = DummyDB()
244 self.db = DummyDB()
216 return
245 return
217
246
218 # use detect_types so that timestamps return datetime objects
247 # use detect_types so that timestamps return datetime objects
219 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
248 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
220 kwargs.update(self.connection_options)
249 kwargs.update(self.connection_options)
221 self.db = sqlite3.connect(self.hist_file, **kwargs)
250 self.db = sqlite3.connect(self.hist_file, **kwargs)
222 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
251 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
223 primary key autoincrement, start timestamp,
252 primary key autoincrement, start timestamp,
224 end timestamp, num_cmds integer, remark text)""")
253 end timestamp, num_cmds integer, remark text)""")
225 self.db.execute("""CREATE TABLE IF NOT EXISTS history
254 self.db.execute("""CREATE TABLE IF NOT EXISTS history
226 (session integer, line integer, source text, source_raw text,
255 (session integer, line integer, source text, source_raw text,
227 PRIMARY KEY (session, line))""")
256 PRIMARY KEY (session, line))""")
228 # Output history is optional, but ensure the table's there so it can be
257 # Output history is optional, but ensure the table's there so it can be
229 # enabled later.
258 # enabled later.
230 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
259 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
231 (session integer, line integer, output text,
260 (session integer, line integer, output text,
232 PRIMARY KEY (session, line))""")
261 PRIMARY KEY (session, line))""")
233 self.db.commit()
262 self.db.commit()
263 # success! reset corrupt db count
264 self._corrupt_db_counter = 0
234
265
235 def writeout_cache(self):
266 def writeout_cache(self):
236 """Overridden by HistoryManager to dump the cache before certain
267 """Overridden by HistoryManager to dump the cache before certain
237 database lookups."""
268 database lookups."""
238 pass
269 pass
239
270
240 ## -------------------------------
271 ## -------------------------------
241 ## Methods for retrieving history:
272 ## Methods for retrieving history:
242 ## -------------------------------
273 ## -------------------------------
243 def _run_sql(self, sql, params, raw=True, output=False):
274 def _run_sql(self, sql, params, raw=True, output=False):
244 """Prepares and runs an SQL query for the history database.
275 """Prepares and runs an SQL query for the history database.
245
276
246 Parameters
277 Parameters
247 ----------
278 ----------
248 sql : str
279 sql : str
249 Any filtering expressions to go after SELECT ... FROM ...
280 Any filtering expressions to go after SELECT ... FROM ...
250 params : tuple
281 params : tuple
251 Parameters passed to the SQL query (to replace "?")
282 Parameters passed to the SQL query (to replace "?")
252 raw, output : bool
283 raw, output : bool
253 See :meth:`get_range`
284 See :meth:`get_range`
254
285
255 Returns
286 Returns
256 -------
287 -------
257 Tuples as :meth:`get_range`
288 Tuples as :meth:`get_range`
258 """
289 """
259 toget = 'source_raw' if raw else 'source'
290 toget = 'source_raw' if raw else 'source'
260 sqlfrom = "history"
291 sqlfrom = "history"
261 if output:
292 if output:
262 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
293 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
263 toget = "history.%s, output_history.output" % toget
294 toget = "history.%s, output_history.output" % toget
264 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
295 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
265 (toget, sqlfrom) + sql, params)
296 (toget, sqlfrom) + sql, params)
266 if output: # Regroup into 3-tuples, and parse JSON
297 if output: # Regroup into 3-tuples, and parse JSON
267 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
298 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
268 return cur
299 return cur
269
300
270 @needs_sqlite
301 @needs_sqlite
271 @catch_corrupt_db
302 @catch_corrupt_db
272 def get_session_info(self, session):
303 def get_session_info(self, session):
273 """Get info about a session.
304 """Get info about a session.
274
305
275 Parameters
306 Parameters
276 ----------
307 ----------
277
308
278 session : int
309 session : int
279 Session number to retrieve.
310 Session number to retrieve.
280
311
281 Returns
312 Returns
282 -------
313 -------
283
314
284 session_id : int
315 session_id : int
285 Session ID number
316 Session ID number
286 start : datetime
317 start : datetime
287 Timestamp for the start of the session.
318 Timestamp for the start of the session.
288 end : datetime
319 end : datetime
289 Timestamp for the end of the session, or None if IPython crashed.
320 Timestamp for the end of the session, or None if IPython crashed.
290 num_cmds : int
321 num_cmds : int
291 Number of commands run, or None if IPython crashed.
322 Number of commands run, or None if IPython crashed.
292 remark : unicode
323 remark : unicode
293 A manually set description.
324 A manually set description.
294 """
325 """
295 query = "SELECT * from sessions where session == ?"
326 query = "SELECT * from sessions where session == ?"
296 return self.db.execute(query, (session,)).fetchone()
327 return self.db.execute(query, (session,)).fetchone()
297
328
298 @catch_corrupt_db
329 @catch_corrupt_db
299 def get_last_session_id(self):
330 def get_last_session_id(self):
300 """Get the last session ID currently in the database.
331 """Get the last session ID currently in the database.
301
332
302 Within IPython, this should be the same as the value stored in
333 Within IPython, this should be the same as the value stored in
303 :attr:`HistoryManager.session_number`.
334 :attr:`HistoryManager.session_number`.
304 """
335 """
305 for record in self.get_tail(n=1, include_latest=True):
336 for record in self.get_tail(n=1, include_latest=True):
306 return record[0]
337 return record[0]
307
338
308 @catch_corrupt_db
339 @catch_corrupt_db
309 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
340 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
310 """Get the last n lines from the history database.
341 """Get the last n lines from the history database.
311
342
312 Parameters
343 Parameters
313 ----------
344 ----------
314 n : int
345 n : int
315 The number of lines to get
346 The number of lines to get
316 raw, output : bool
347 raw, output : bool
317 See :meth:`get_range`
348 See :meth:`get_range`
318 include_latest : bool
349 include_latest : bool
319 If False (default), n+1 lines are fetched, and the latest one
350 If False (default), n+1 lines are fetched, and the latest one
320 is discarded. This is intended to be used where the function
351 is discarded. This is intended to be used where the function
321 is called by a user command, which it should not return.
352 is called by a user command, which it should not return.
322
353
323 Returns
354 Returns
324 -------
355 -------
325 Tuples as :meth:`get_range`
356 Tuples as :meth:`get_range`
326 """
357 """
327 self.writeout_cache()
358 self.writeout_cache()
328 if not include_latest:
359 if not include_latest:
329 n += 1
360 n += 1
330 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
361 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
331 (n,), raw=raw, output=output)
362 (n,), raw=raw, output=output)
332 if not include_latest:
363 if not include_latest:
333 return reversed(list(cur)[1:])
364 return reversed(list(cur)[1:])
334 return reversed(list(cur))
365 return reversed(list(cur))
335
366
336 @catch_corrupt_db
367 @catch_corrupt_db
337 def search(self, pattern="*", raw=True, search_raw=True,
368 def search(self, pattern="*", raw=True, search_raw=True,
338 output=False, n=None, unique=False):
369 output=False, n=None, unique=False):
339 """Search the database using unix glob-style matching (wildcards
370 """Search the database using unix glob-style matching (wildcards
340 * and ?).
371 * and ?).
341
372
342 Parameters
373 Parameters
343 ----------
374 ----------
344 pattern : str
375 pattern : str
345 The wildcarded pattern to match when searching
376 The wildcarded pattern to match when searching
346 search_raw : bool
377 search_raw : bool
347 If True, search the raw input, otherwise, the parsed input
378 If True, search the raw input, otherwise, the parsed input
348 raw, output : bool
379 raw, output : bool
349 See :meth:`get_range`
380 See :meth:`get_range`
350 n : None or int
381 n : None or int
351 If an integer is given, it defines the limit of
382 If an integer is given, it defines the limit of
352 returned entries.
383 returned entries.
353 unique : bool
384 unique : bool
354 When it is true, return only unique entries.
385 When it is true, return only unique entries.
355
386
356 Returns
387 Returns
357 -------
388 -------
358 Tuples as :meth:`get_range`
389 Tuples as :meth:`get_range`
359 """
390 """
360 tosearch = "source_raw" if search_raw else "source"
391 tosearch = "source_raw" if search_raw else "source"
361 if output:
392 if output:
362 tosearch = "history." + tosearch
393 tosearch = "history." + tosearch
363 self.writeout_cache()
394 self.writeout_cache()
364 sqlform = "WHERE %s GLOB ?" % tosearch
395 sqlform = "WHERE %s GLOB ?" % tosearch
365 params = (pattern,)
396 params = (pattern,)
366 if unique:
397 if unique:
367 sqlform += ' GROUP BY {0}'.format(tosearch)
398 sqlform += ' GROUP BY {0}'.format(tosearch)
368 if n is not None:
399 if n is not None:
369 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
400 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
370 params += (n,)
401 params += (n,)
371 elif unique:
402 elif unique:
372 sqlform += " ORDER BY session, line"
403 sqlform += " ORDER BY session, line"
373 cur = self._run_sql(sqlform, params, raw=raw, output=output)
404 cur = self._run_sql(sqlform, params, raw=raw, output=output)
374 if n is not None:
405 if n is not None:
375 return reversed(list(cur))
406 return reversed(list(cur))
376 return cur
407 return cur
377
408
378 @catch_corrupt_db
409 @catch_corrupt_db
379 def get_range(self, session, start=1, stop=None, raw=True,output=False):
410 def get_range(self, session, start=1, stop=None, raw=True,output=False):
380 """Retrieve input by session.
411 """Retrieve input by session.
381
412
382 Parameters
413 Parameters
383 ----------
414 ----------
384 session : int
415 session : int
385 Session number to retrieve.
416 Session number to retrieve.
386 start : int
417 start : int
387 First line to retrieve.
418 First line to retrieve.
388 stop : int
419 stop : int
389 End of line range (excluded from output itself). If None, retrieve
420 End of line range (excluded from output itself). If None, retrieve
390 to the end of the session.
421 to the end of the session.
391 raw : bool
422 raw : bool
392 If True, return untranslated input
423 If True, return untranslated input
393 output : bool
424 output : bool
394 If True, attempt to include output. This will be 'real' Python
425 If True, attempt to include output. This will be 'real' Python
395 objects for the current session, or text reprs from previous
426 objects for the current session, or text reprs from previous
396 sessions if db_log_output was enabled at the time. Where no output
427 sessions if db_log_output was enabled at the time. Where no output
397 is found, None is used.
428 is found, None is used.
398
429
399 Returns
430 Returns
400 -------
431 -------
401 entries
432 entries
402 An iterator over the desired lines. Each line is a 3-tuple, either
433 An iterator over the desired lines. Each line is a 3-tuple, either
403 (session, line, input) if output is False, or
434 (session, line, input) if output is False, or
404 (session, line, (input, output)) if output is True.
435 (session, line, (input, output)) if output is True.
405 """
436 """
406 if stop:
437 if stop:
407 lineclause = "line >= ? AND line < ?"
438 lineclause = "line >= ? AND line < ?"
408 params = (session, start, stop)
439 params = (session, start, stop)
409 else:
440 else:
410 lineclause = "line>=?"
441 lineclause = "line>=?"
411 params = (session, start)
442 params = (session, start)
412
443
413 return self._run_sql("WHERE session==? AND %s" % lineclause,
444 return self._run_sql("WHERE session==? AND %s" % lineclause,
414 params, raw=raw, output=output)
445 params, raw=raw, output=output)
415
446
416 def get_range_by_str(self, rangestr, raw=True, output=False):
447 def get_range_by_str(self, rangestr, raw=True, output=False):
417 """Get lines of history from a string of ranges, as used by magic
448 """Get lines of history from a string of ranges, as used by magic
418 commands %hist, %save, %macro, etc.
449 commands %hist, %save, %macro, etc.
419
450
420 Parameters
451 Parameters
421 ----------
452 ----------
422 rangestr : str
453 rangestr : str
423 A string specifying ranges, e.g. "5 ~2/1-4". See
454 A string specifying ranges, e.g. "5 ~2/1-4". See
424 :func:`magic_history` for full details.
455 :func:`magic_history` for full details.
425 raw, output : bool
456 raw, output : bool
426 As :meth:`get_range`
457 As :meth:`get_range`
427
458
428 Returns
459 Returns
429 -------
460 -------
430 Tuples as :meth:`get_range`
461 Tuples as :meth:`get_range`
431 """
462 """
432 for sess, s, e in extract_hist_ranges(rangestr):
463 for sess, s, e in extract_hist_ranges(rangestr):
433 for line in self.get_range(sess, s, e, raw=raw, output=output):
464 for line in self.get_range(sess, s, e, raw=raw, output=output):
434 yield line
465 yield line
435
466
436
467
437 class HistoryManager(HistoryAccessor):
468 class HistoryManager(HistoryAccessor):
438 """A class to organize all history-related functionality in one place.
469 """A class to organize all history-related functionality in one place.
439 """
470 """
440 # Public interface
471 # Public interface
441
472
442 # An instance of the IPython shell we are attached to
473 # An instance of the IPython shell we are attached to
443 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
474 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
444 allow_none=True)
475 allow_none=True)
445 # Lists to hold processed and raw history. These start with a blank entry
476 # Lists to hold processed and raw history. These start with a blank entry
446 # so that we can index them starting from 1
477 # so that we can index them starting from 1
447 input_hist_parsed = List([""])
478 input_hist_parsed = List([""])
448 input_hist_raw = List([""])
479 input_hist_raw = List([""])
449 # A list of directories visited during session
480 # A list of directories visited during session
450 dir_hist = List()
481 dir_hist = List()
451 def _dir_hist_default(self):
482 def _dir_hist_default(self):
452 try:
483 try:
453 return [py3compat.getcwd()]
484 return [py3compat.getcwd()]
454 except OSError:
485 except OSError:
455 return []
486 return []
456
487
457 # A dict of output history, keyed with ints from the shell's
488 # A dict of output history, keyed with ints from the shell's
458 # execution count.
489 # execution count.
459 output_hist = Dict()
490 output_hist = Dict()
460 # The text/plain repr of outputs.
491 # The text/plain repr of outputs.
461 output_hist_reprs = Dict()
492 output_hist_reprs = Dict()
462
493
463 # The number of the current session in the history database
494 # The number of the current session in the history database
464 session_number = Integer()
495 session_number = Integer()
465
496
466 db_log_output = Bool(False, config=True,
497 db_log_output = Bool(False, config=True,
467 help="Should the history database include output? (default: no)"
498 help="Should the history database include output? (default: no)"
468 )
499 )
469 db_cache_size = Integer(0, config=True,
500 db_cache_size = Integer(0, config=True,
470 help="Write to database every x commands (higher values save disk access & power).\n"
501 help="Write to database every x commands (higher values save disk access & power).\n"
471 "Values of 1 or less effectively disable caching."
502 "Values of 1 or less effectively disable caching."
472 )
503 )
473 # The input and output caches
504 # The input and output caches
474 db_input_cache = List()
505 db_input_cache = List()
475 db_output_cache = List()
506 db_output_cache = List()
476
507
477 # History saving in separate thread
508 # History saving in separate thread
478 save_thread = Instance('IPython.core.history.HistorySavingThread',
509 save_thread = Instance('IPython.core.history.HistorySavingThread',
479 allow_none=True)
510 allow_none=True)
480 try: # Event is a function returning an instance of _Event...
511 try: # Event is a function returning an instance of _Event...
481 save_flag = Instance(threading._Event, allow_none=True)
512 save_flag = Instance(threading._Event, allow_none=True)
482 except AttributeError: # ...until Python 3.3, when it's a class.
513 except AttributeError: # ...until Python 3.3, when it's a class.
483 save_flag = Instance(threading.Event, allow_none=True)
514 save_flag = Instance(threading.Event, allow_none=True)
484
515
485 # Private interface
516 # Private interface
486 # Variables used to store the three last inputs from the user. On each new
517 # Variables used to store the three last inputs from the user. On each new
487 # history update, we populate the user's namespace with these, shifted as
518 # history update, we populate the user's namespace with these, shifted as
488 # necessary.
519 # necessary.
489 _i00 = Unicode(u'')
520 _i00 = Unicode(u'')
490 _i = Unicode(u'')
521 _i = Unicode(u'')
491 _ii = Unicode(u'')
522 _ii = Unicode(u'')
492 _iii = Unicode(u'')
523 _iii = Unicode(u'')
493
524
494 # A regex matching all forms of the exit command, so that we don't store
525 # A regex matching all forms of the exit command, so that we don't store
495 # them in the history (it's annoying to rewind the first entry and land on
526 # them in the history (it's annoying to rewind the first entry and land on
496 # an exit call).
527 # an exit call).
497 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
528 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
498
529
499 def __init__(self, shell=None, config=None, **traits):
530 def __init__(self, shell=None, config=None, **traits):
500 """Create a new history manager associated with a shell instance.
531 """Create a new history manager associated with a shell instance.
501 """
532 """
502 # We need a pointer back to the shell for various tasks.
533 # We need a pointer back to the shell for various tasks.
503 super(HistoryManager, self).__init__(shell=shell, config=config,
534 super(HistoryManager, self).__init__(shell=shell, config=config,
504 **traits)
535 **traits)
505 self.save_flag = threading.Event()
536 self.save_flag = threading.Event()
506 self.db_input_cache_lock = threading.Lock()
537 self.db_input_cache_lock = threading.Lock()
507 self.db_output_cache_lock = threading.Lock()
538 self.db_output_cache_lock = threading.Lock()
508
539
509 try:
540 try:
510 self.new_session()
541 self.new_session()
511 except OperationalError:
542 except OperationalError:
512 self.log.error("Failed to create history session in %s. History will not be saved.",
543 self.log.error("Failed to create history session in %s. History will not be saved.",
513 self.hist_file, exc_info=True)
544 self.hist_file, exc_info=True)
514 self.hist_file = ':memory:'
545 self.hist_file = ':memory:'
515
546
516 if self.enabled and self.hist_file != ':memory:':
547 if self.enabled and self.hist_file != ':memory:':
517 self.save_thread = HistorySavingThread(self)
548 self.save_thread = HistorySavingThread(self)
518 self.save_thread.start()
549 self.save_thread.start()
519
550
520 def _get_hist_file_name(self, profile=None):
551 def _get_hist_file_name(self, profile=None):
521 """Get default history file name based on the Shell's profile.
552 """Get default history file name based on the Shell's profile.
522
553
523 The profile parameter is ignored, but must exist for compatibility with
554 The profile parameter is ignored, but must exist for compatibility with
524 the parent class."""
555 the parent class."""
525 profile_dir = self.shell.profile_dir.location
556 profile_dir = self.shell.profile_dir.location
526 return os.path.join(profile_dir, 'history.sqlite')
557 return os.path.join(profile_dir, 'history.sqlite')
527
558
528 @needs_sqlite
559 @needs_sqlite
529 def new_session(self, conn=None):
560 def new_session(self, conn=None):
530 """Get a new session number."""
561 """Get a new session number."""
531 if conn is None:
562 if conn is None:
532 conn = self.db
563 conn = self.db
533
564
534 with conn:
565 with conn:
535 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
566 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
536 NULL, "") """, (datetime.datetime.now(),))
567 NULL, "") """, (datetime.datetime.now(),))
537 self.session_number = cur.lastrowid
568 self.session_number = cur.lastrowid
538
569
539 def end_session(self):
570 def end_session(self):
540 """Close the database session, filling in the end time and line count."""
571 """Close the database session, filling in the end time and line count."""
541 self.writeout_cache()
572 self.writeout_cache()
542 with self.db:
573 with self.db:
543 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
574 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
544 session==?""", (datetime.datetime.now(),
575 session==?""", (datetime.datetime.now(),
545 len(self.input_hist_parsed)-1, self.session_number))
576 len(self.input_hist_parsed)-1, self.session_number))
546 self.session_number = 0
577 self.session_number = 0
547
578
548 def name_session(self, name):
579 def name_session(self, name):
549 """Give the current session a name in the history database."""
580 """Give the current session a name in the history database."""
550 with self.db:
581 with self.db:
551 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
582 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
552 (name, self.session_number))
583 (name, self.session_number))
553
584
554 def reset(self, new_session=True):
585 def reset(self, new_session=True):
555 """Clear the session history, releasing all object references, and
586 """Clear the session history, releasing all object references, and
556 optionally open a new session."""
587 optionally open a new session."""
557 self.output_hist.clear()
588 self.output_hist.clear()
558 # The directory history can't be completely empty
589 # The directory history can't be completely empty
559 self.dir_hist[:] = [py3compat.getcwd()]
590 self.dir_hist[:] = [py3compat.getcwd()]
560
591
561 if new_session:
592 if new_session:
562 if self.session_number:
593 if self.session_number:
563 self.end_session()
594 self.end_session()
564 self.input_hist_parsed[:] = [""]
595 self.input_hist_parsed[:] = [""]
565 self.input_hist_raw[:] = [""]
596 self.input_hist_raw[:] = [""]
566 self.new_session()
597 self.new_session()
567
598
568 # ------------------------------
599 # ------------------------------
569 # Methods for retrieving history
600 # Methods for retrieving history
570 # ------------------------------
601 # ------------------------------
571 def get_session_info(self, session=0):
602 def get_session_info(self, session=0):
572 """Get info about a session.
603 """Get info about a session.
573
604
574 Parameters
605 Parameters
575 ----------
606 ----------
576
607
577 session : int
608 session : int
578 Session number to retrieve. The current session is 0, and negative
609 Session number to retrieve. The current session is 0, and negative
579 numbers count back from current session, so -1 is the previous session.
610 numbers count back from current session, so -1 is the previous session.
580
611
581 Returns
612 Returns
582 -------
613 -------
583
614
584 session_id : int
615 session_id : int
585 Session ID number
616 Session ID number
586 start : datetime
617 start : datetime
587 Timestamp for the start of the session.
618 Timestamp for the start of the session.
588 end : datetime
619 end : datetime
589 Timestamp for the end of the session, or None if IPython crashed.
620 Timestamp for the end of the session, or None if IPython crashed.
590 num_cmds : int
621 num_cmds : int
591 Number of commands run, or None if IPython crashed.
622 Number of commands run, or None if IPython crashed.
592 remark : unicode
623 remark : unicode
593 A manually set description.
624 A manually set description.
594 """
625 """
595 if session <= 0:
626 if session <= 0:
596 session += self.session_number
627 session += self.session_number
597
628
598 return super(HistoryManager, self).get_session_info(session=session)
629 return super(HistoryManager, self).get_session_info(session=session)
599
630
600 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
631 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
601 """Get input and output history from the current session. Called by
632 """Get input and output history from the current session. Called by
602 get_range, and takes similar parameters."""
633 get_range, and takes similar parameters."""
603 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
634 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
604
635
605 n = len(input_hist)
636 n = len(input_hist)
606 if start < 0:
637 if start < 0:
607 start += n
638 start += n
608 if not stop or (stop > n):
639 if not stop or (stop > n):
609 stop = n
640 stop = n
610 elif stop < 0:
641 elif stop < 0:
611 stop += n
642 stop += n
612
643
613 for i in range(start, stop):
644 for i in range(start, stop):
614 if output:
645 if output:
615 line = (input_hist[i], self.output_hist_reprs.get(i))
646 line = (input_hist[i], self.output_hist_reprs.get(i))
616 else:
647 else:
617 line = input_hist[i]
648 line = input_hist[i]
618 yield (0, i, line)
649 yield (0, i, line)
619
650
620 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
651 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
621 """Retrieve input by session.
652 """Retrieve input by session.
622
653
623 Parameters
654 Parameters
624 ----------
655 ----------
625 session : int
656 session : int
626 Session number to retrieve. The current session is 0, and negative
657 Session number to retrieve. The current session is 0, and negative
627 numbers count back from current session, so -1 is previous session.
658 numbers count back from current session, so -1 is previous session.
628 start : int
659 start : int
629 First line to retrieve.
660 First line to retrieve.
630 stop : int
661 stop : int
631 End of line range (excluded from output itself). If None, retrieve
662 End of line range (excluded from output itself). If None, retrieve
632 to the end of the session.
663 to the end of the session.
633 raw : bool
664 raw : bool
634 If True, return untranslated input
665 If True, return untranslated input
635 output : bool
666 output : bool
636 If True, attempt to include output. This will be 'real' Python
667 If True, attempt to include output. This will be 'real' Python
637 objects for the current session, or text reprs from previous
668 objects for the current session, or text reprs from previous
638 sessions if db_log_output was enabled at the time. Where no output
669 sessions if db_log_output was enabled at the time. Where no output
639 is found, None is used.
670 is found, None is used.
640
671
641 Returns
672 Returns
642 -------
673 -------
643 entries
674 entries
644 An iterator over the desired lines. Each line is a 3-tuple, either
675 An iterator over the desired lines. Each line is a 3-tuple, either
645 (session, line, input) if output is False, or
676 (session, line, input) if output is False, or
646 (session, line, (input, output)) if output is True.
677 (session, line, (input, output)) if output is True.
647 """
678 """
648 if session <= 0:
679 if session <= 0:
649 session += self.session_number
680 session += self.session_number
650 if session==self.session_number: # Current session
681 if session==self.session_number: # Current session
651 return self._get_range_session(start, stop, raw, output)
682 return self._get_range_session(start, stop, raw, output)
652 return super(HistoryManager, self).get_range(session, start, stop, raw,
683 return super(HistoryManager, self).get_range(session, start, stop, raw,
653 output)
684 output)
654
685
655 ## ----------------------------
686 ## ----------------------------
656 ## Methods for storing history:
687 ## Methods for storing history:
657 ## ----------------------------
688 ## ----------------------------
658 def store_inputs(self, line_num, source, source_raw=None):
689 def store_inputs(self, line_num, source, source_raw=None):
659 """Store source and raw input in history and create input cache
690 """Store source and raw input in history and create input cache
660 variables ``_i*``.
691 variables ``_i*``.
661
692
662 Parameters
693 Parameters
663 ----------
694 ----------
664 line_num : int
695 line_num : int
665 The prompt number of this input.
696 The prompt number of this input.
666
697
667 source : str
698 source : str
668 Python input.
699 Python input.
669
700
670 source_raw : str, optional
701 source_raw : str, optional
671 If given, this is the raw input without any IPython transformations
702 If given, this is the raw input without any IPython transformations
672 applied to it. If not given, ``source`` is used.
703 applied to it. If not given, ``source`` is used.
673 """
704 """
674 if source_raw is None:
705 if source_raw is None:
675 source_raw = source
706 source_raw = source
676 source = source.rstrip('\n')
707 source = source.rstrip('\n')
677 source_raw = source_raw.rstrip('\n')
708 source_raw = source_raw.rstrip('\n')
678
709
679 # do not store exit/quit commands
710 # do not store exit/quit commands
680 if self._exit_re.match(source_raw.strip()):
711 if self._exit_re.match(source_raw.strip()):
681 return
712 return
682
713
683 self.input_hist_parsed.append(source)
714 self.input_hist_parsed.append(source)
684 self.input_hist_raw.append(source_raw)
715 self.input_hist_raw.append(source_raw)
685
716
686 with self.db_input_cache_lock:
717 with self.db_input_cache_lock:
687 self.db_input_cache.append((line_num, source, source_raw))
718 self.db_input_cache.append((line_num, source, source_raw))
688 # Trigger to flush cache and write to DB.
719 # Trigger to flush cache and write to DB.
689 if len(self.db_input_cache) >= self.db_cache_size:
720 if len(self.db_input_cache) >= self.db_cache_size:
690 self.save_flag.set()
721 self.save_flag.set()
691
722
692 # update the auto _i variables
723 # update the auto _i variables
693 self._iii = self._ii
724 self._iii = self._ii
694 self._ii = self._i
725 self._ii = self._i
695 self._i = self._i00
726 self._i = self._i00
696 self._i00 = source_raw
727 self._i00 = source_raw
697
728
698 # hackish access to user namespace to create _i1,_i2... dynamically
729 # hackish access to user namespace to create _i1,_i2... dynamically
699 new_i = '_i%s' % line_num
730 new_i = '_i%s' % line_num
700 to_main = {'_i': self._i,
731 to_main = {'_i': self._i,
701 '_ii': self._ii,
732 '_ii': self._ii,
702 '_iii': self._iii,
733 '_iii': self._iii,
703 new_i : self._i00 }
734 new_i : self._i00 }
704
735
705 if self.shell is not None:
736 if self.shell is not None:
706 self.shell.push(to_main, interactive=False)
737 self.shell.push(to_main, interactive=False)
707
738
708 def store_output(self, line_num):
739 def store_output(self, line_num):
709 """If database output logging is enabled, this saves all the
740 """If database output logging is enabled, this saves all the
710 outputs from the indicated prompt number to the database. It's
741 outputs from the indicated prompt number to the database. It's
711 called by run_cell after code has been executed.
742 called by run_cell after code has been executed.
712
743
713 Parameters
744 Parameters
714 ----------
745 ----------
715 line_num : int
746 line_num : int
716 The line number from which to save outputs
747 The line number from which to save outputs
717 """
748 """
718 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
749 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
719 return
750 return
720 output = self.output_hist_reprs[line_num]
751 output = self.output_hist_reprs[line_num]
721
752
722 with self.db_output_cache_lock:
753 with self.db_output_cache_lock:
723 self.db_output_cache.append((line_num, output))
754 self.db_output_cache.append((line_num, output))
724 if self.db_cache_size <= 1:
755 if self.db_cache_size <= 1:
725 self.save_flag.set()
756 self.save_flag.set()
726
757
727 def _writeout_input_cache(self, conn):
758 def _writeout_input_cache(self, conn):
728 with conn:
759 with conn:
729 for line in self.db_input_cache:
760 for line in self.db_input_cache:
730 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
761 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
731 (self.session_number,)+line)
762 (self.session_number,)+line)
732
763
733 def _writeout_output_cache(self, conn):
764 def _writeout_output_cache(self, conn):
734 with conn:
765 with conn:
735 for line in self.db_output_cache:
766 for line in self.db_output_cache:
736 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
767 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
737 (self.session_number,)+line)
768 (self.session_number,)+line)
738
769
739 @needs_sqlite
770 @needs_sqlite
740 def writeout_cache(self, conn=None):
771 def writeout_cache(self, conn=None):
741 """Write any entries in the cache to the database."""
772 """Write any entries in the cache to the database."""
742 if conn is None:
773 if conn is None:
743 conn = self.db
774 conn = self.db
744
775
745 with self.db_input_cache_lock:
776 with self.db_input_cache_lock:
746 try:
777 try:
747 self._writeout_input_cache(conn)
778 self._writeout_input_cache(conn)
748 except sqlite3.IntegrityError:
779 except sqlite3.IntegrityError:
749 self.new_session(conn)
780 self.new_session(conn)
750 print("ERROR! Session/line number was not unique in",
781 print("ERROR! Session/line number was not unique in",
751 "database. History logging moved to new session",
782 "database. History logging moved to new session",
752 self.session_number)
783 self.session_number)
753 try:
784 try:
754 # Try writing to the new session. If this fails, don't
785 # Try writing to the new session. If this fails, don't
755 # recurse
786 # recurse
756 self._writeout_input_cache(conn)
787 self._writeout_input_cache(conn)
757 except sqlite3.IntegrityError:
788 except sqlite3.IntegrityError:
758 pass
789 pass
759 finally:
790 finally:
760 self.db_input_cache = []
791 self.db_input_cache = []
761
792
762 with self.db_output_cache_lock:
793 with self.db_output_cache_lock:
763 try:
794 try:
764 self._writeout_output_cache(conn)
795 self._writeout_output_cache(conn)
765 except sqlite3.IntegrityError:
796 except sqlite3.IntegrityError:
766 print("!! Session/line number for output was not unique",
797 print("!! Session/line number for output was not unique",
767 "in database. Output will not be stored.")
798 "in database. Output will not be stored.")
768 finally:
799 finally:
769 self.db_output_cache = []
800 self.db_output_cache = []
770
801
771
802
772 class HistorySavingThread(threading.Thread):
803 class HistorySavingThread(threading.Thread):
773 """This thread takes care of writing history to the database, so that
804 """This thread takes care of writing history to the database, so that
774 the UI isn't held up while that happens.
805 the UI isn't held up while that happens.
775
806
776 It waits for the HistoryManager's save_flag to be set, then writes out
807 It waits for the HistoryManager's save_flag to be set, then writes out
777 the history cache. The main thread is responsible for setting the flag when
808 the history cache. The main thread is responsible for setting the flag when
778 the cache size reaches a defined threshold."""
809 the cache size reaches a defined threshold."""
779 daemon = True
810 daemon = True
780 stop_now = False
811 stop_now = False
781 enabled = True
812 enabled = True
782 def __init__(self, history_manager):
813 def __init__(self, history_manager):
783 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
814 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
784 self.history_manager = history_manager
815 self.history_manager = history_manager
785 self.enabled = history_manager.enabled
816 self.enabled = history_manager.enabled
786 atexit.register(self.stop)
817 atexit.register(self.stop)
787
818
788 @needs_sqlite
819 @needs_sqlite
789 def run(self):
820 def run(self):
790 # We need a separate db connection per thread:
821 # We need a separate db connection per thread:
791 try:
822 try:
792 self.db = sqlite3.connect(self.history_manager.hist_file,
823 self.db = sqlite3.connect(self.history_manager.hist_file,
793 **self.history_manager.connection_options
824 **self.history_manager.connection_options
794 )
825 )
795 while True:
826 while True:
796 self.history_manager.save_flag.wait()
827 self.history_manager.save_flag.wait()
797 if self.stop_now:
828 if self.stop_now:
798 self.db.close()
829 self.db.close()
799 return
830 return
800 self.history_manager.save_flag.clear()
831 self.history_manager.save_flag.clear()
801 self.history_manager.writeout_cache(self.db)
832 self.history_manager.writeout_cache(self.db)
802 except Exception as e:
833 except Exception as e:
803 print(("The history saving thread hit an unexpected error (%s)."
834 print(("The history saving thread hit an unexpected error (%s)."
804 "History will not be written to the database.") % repr(e))
835 "History will not be written to the database.") % repr(e))
805
836
806 def stop(self):
837 def stop(self):
807 """This can be called from the main thread to safely stop this thread.
838 """This can be called from the main thread to safely stop this thread.
808
839
809 Note that it does not attempt to write out remaining history before
840 Note that it does not attempt to write out remaining history before
810 exiting. That should be done by calling the HistoryManager's
841 exiting. That should be done by calling the HistoryManager's
811 end_session method."""
842 end_session method."""
812 self.stop_now = True
843 self.stop_now = True
813 self.history_manager.save_flag.set()
844 self.history_manager.save_flag.set()
814 self.join()
845 self.join()
815
846
816
847
817 # To match, e.g. ~5/8-~2/3
848 # To match, e.g. ~5/8-~2/3
818 range_re = re.compile(r"""
849 range_re = re.compile(r"""
819 ((?P<startsess>~?\d+)/)?
850 ((?P<startsess>~?\d+)/)?
820 (?P<start>\d+)?
851 (?P<start>\d+)?
821 ((?P<sep>[\-:])
852 ((?P<sep>[\-:])
822 ((?P<endsess>~?\d+)/)?
853 ((?P<endsess>~?\d+)/)?
823 (?P<end>\d+))?
854 (?P<end>\d+))?
824 $""", re.VERBOSE)
855 $""", re.VERBOSE)
825
856
826
857
827 def extract_hist_ranges(ranges_str):
858 def extract_hist_ranges(ranges_str):
828 """Turn a string of history ranges into 3-tuples of (session, start, stop).
859 """Turn a string of history ranges into 3-tuples of (session, start, stop).
829
860
830 Examples
861 Examples
831 --------
862 --------
832 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
863 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
833 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
864 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
834 """
865 """
835 for range_str in ranges_str.split():
866 for range_str in ranges_str.split():
836 rmatch = range_re.match(range_str)
867 rmatch = range_re.match(range_str)
837 if not rmatch:
868 if not rmatch:
838 continue
869 continue
839 start = rmatch.group("start")
870 start = rmatch.group("start")
840 if start:
871 if start:
841 start = int(start)
872 start = int(start)
842 end = rmatch.group("end")
873 end = rmatch.group("end")
843 # If no end specified, get (a, a + 1)
874 # If no end specified, get (a, a + 1)
844 end = int(end) if end else start + 1
875 end = int(end) if end else start + 1
845 else: # start not specified
876 else: # start not specified
846 if not rmatch.group('startsess'): # no startsess
877 if not rmatch.group('startsess'): # no startsess
847 continue
878 continue
848 start = 1
879 start = 1
849 end = None # provide the entire session hist
880 end = None # provide the entire session hist
850
881
851 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
882 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
852 end += 1
883 end += 1
853 startsess = rmatch.group("startsess") or "0"
884 startsess = rmatch.group("startsess") or "0"
854 endsess = rmatch.group("endsess") or startsess
885 endsess = rmatch.group("endsess") or startsess
855 startsess = int(startsess.replace("~","-"))
886 startsess = int(startsess.replace("~","-"))
856 endsess = int(endsess.replace("~","-"))
887 endsess = int(endsess.replace("~","-"))
857 assert endsess >= startsess, "start session must be earlier than end session"
888 assert endsess >= startsess, "start session must be earlier than end session"
858
889
859 if endsess == startsess:
890 if endsess == startsess:
860 yield (startsess, start, end)
891 yield (startsess, start, end)
861 continue
892 continue
862 # Multiple sessions in one range:
893 # Multiple sessions in one range:
863 yield (startsess, start, None)
894 yield (startsess, start, None)
864 for sess in range(startsess+1, endsess):
895 for sess in range(startsess+1, endsess):
865 yield (sess, 1, None)
896 yield (sess, 1, None)
866 yield (endsess, 1, end)
897 yield (endsess, 1, end)
867
898
868
899
869 def _format_lineno(session, line):
900 def _format_lineno(session, line):
870 """Helper function to format line numbers properly."""
901 """Helper function to format line numbers properly."""
871 if session == 0:
902 if session == 0:
872 return str(line)
903 return str(line)
873 return "%s#%s" % (session, line)
904 return "%s#%s" % (session, line)
General Comments 0
You need to be logged in to leave comments. Login now