##// END OF EJS Templates
Remove python2 specific code
Srinivas Reddy Thatiparthy -
Show More
@@ -1,910 +1,907 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6
6
7 import atexit
7 import atexit
8 import datetime
8 import datetime
9 import os
9 import os
10 import re
10 import re
11 try:
11 try:
12 import sqlite3
12 import sqlite3
13 except ImportError:
13 except ImportError:
14 try:
14 try:
15 from pysqlite2 import dbapi2 as sqlite3
15 from pysqlite2 import dbapi2 as sqlite3
16 except ImportError:
16 except ImportError:
17 sqlite3 = None
17 sqlite3 = None
18 import threading
18 import threading
19
19
20 from traitlets.config.configurable import LoggingConfigurable
20 from traitlets.config.configurable import LoggingConfigurable
21 from decorator import decorator
21 from decorator import decorator
22 from IPython.utils.decorators import undoc
22 from IPython.utils.decorators import undoc
23 from IPython.utils.path import locate_profile
23 from IPython.utils.path import locate_profile
24 from IPython.utils import py3compat
24 from IPython.utils import py3compat
25 from traitlets import (
25 from traitlets import (
26 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
26 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
27 default, observe,
27 default, observe,
28 )
28 )
29 from warnings import warn
29 from warnings import warn
30
30
31 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
32 # Classes and functions
32 # Classes and functions
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34
34
35 @undoc
35 @undoc
36 class DummyDB(object):
36 class DummyDB(object):
37 """Dummy DB that will act as a black hole for history.
37 """Dummy DB that will act as a black hole for history.
38
38
39 Only used in the absence of sqlite"""
39 Only used in the absence of sqlite"""
40 def execute(*args, **kwargs):
40 def execute(*args, **kwargs):
41 return []
41 return []
42
42
43 def commit(self, *args, **kwargs):
43 def commit(self, *args, **kwargs):
44 pass
44 pass
45
45
46 def __enter__(self, *args, **kwargs):
46 def __enter__(self, *args, **kwargs):
47 pass
47 pass
48
48
49 def __exit__(self, *args, **kwargs):
49 def __exit__(self, *args, **kwargs):
50 pass
50 pass
51
51
52
52
53 @decorator
53 @decorator
54 def needs_sqlite(f, self, *a, **kw):
54 def needs_sqlite(f, self, *a, **kw):
55 """Decorator: return an empty list in the absence of sqlite."""
55 """Decorator: return an empty list in the absence of sqlite."""
56 if sqlite3 is None or not self.enabled:
56 if sqlite3 is None or not self.enabled:
57 return []
57 return []
58 else:
58 else:
59 return f(self, *a, **kw)
59 return f(self, *a, **kw)
60
60
61
61
62 if sqlite3 is not None:
62 if sqlite3 is not None:
63 DatabaseError = sqlite3.DatabaseError
63 DatabaseError = sqlite3.DatabaseError
64 OperationalError = sqlite3.OperationalError
64 OperationalError = sqlite3.OperationalError
65 else:
65 else:
66 @undoc
66 @undoc
67 class DatabaseError(Exception):
67 class DatabaseError(Exception):
68 "Dummy exception when sqlite could not be imported. Should never occur."
68 "Dummy exception when sqlite could not be imported. Should never occur."
69
69
70 @undoc
70 @undoc
71 class OperationalError(Exception):
71 class OperationalError(Exception):
72 "Dummy exception when sqlite could not be imported. Should never occur."
72 "Dummy exception when sqlite could not be imported. Should never occur."
73
73
74 # use 16kB as threshold for whether a corrupt history db should be saved
74 # use 16kB as threshold for whether a corrupt history db should be saved
75 # that should be at least 100 entries or so
75 # that should be at least 100 entries or so
76 _SAVE_DB_SIZE = 16384
76 _SAVE_DB_SIZE = 16384
77
77
78 @decorator
78 @decorator
79 def catch_corrupt_db(f, self, *a, **kw):
79 def catch_corrupt_db(f, self, *a, **kw):
80 """A decorator which wraps HistoryAccessor method calls to catch errors from
80 """A decorator which wraps HistoryAccessor method calls to catch errors from
81 a corrupt SQLite database, move the old database out of the way, and create
81 a corrupt SQLite database, move the old database out of the way, and create
82 a new one.
82 a new one.
83
83
84 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
84 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
85 not just a corrupt file.
85 not just a corrupt file.
86 """
86 """
87 try:
87 try:
88 return f(self, *a, **kw)
88 return f(self, *a, **kw)
89 except (DatabaseError, OperationalError) as e:
89 except (DatabaseError, OperationalError) as e:
90 self._corrupt_db_counter += 1
90 self._corrupt_db_counter += 1
91 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
91 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
92 if self.hist_file != ':memory:':
92 if self.hist_file != ':memory:':
93 if self._corrupt_db_counter > self._corrupt_db_limit:
93 if self._corrupt_db_counter > self._corrupt_db_limit:
94 self.hist_file = ':memory:'
94 self.hist_file = ':memory:'
95 self.log.error("Failed to load history too many times, history will not be saved.")
95 self.log.error("Failed to load history too many times, history will not be saved.")
96 elif os.path.isfile(self.hist_file):
96 elif os.path.isfile(self.hist_file):
97 # move the file out of the way
97 # move the file out of the way
98 base, ext = os.path.splitext(self.hist_file)
98 base, ext = os.path.splitext(self.hist_file)
99 size = os.stat(self.hist_file).st_size
99 size = os.stat(self.hist_file).st_size
100 if size >= _SAVE_DB_SIZE:
100 if size >= _SAVE_DB_SIZE:
101 # if there's significant content, avoid clobbering
101 # if there's significant content, avoid clobbering
102 now = datetime.datetime.now().isoformat().replace(':', '.')
102 now = datetime.datetime.now().isoformat().replace(':', '.')
103 newpath = base + '-corrupt-' + now + ext
103 newpath = base + '-corrupt-' + now + ext
104 # don't clobber previous corrupt backups
104 # don't clobber previous corrupt backups
105 for i in range(100):
105 for i in range(100):
106 if not os.path.isfile(newpath):
106 if not os.path.isfile(newpath):
107 break
107 break
108 else:
108 else:
109 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
109 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
110 else:
110 else:
111 # not much content, possibly empty; don't worry about clobbering
111 # not much content, possibly empty; don't worry about clobbering
112 # maybe we should just delete it?
112 # maybe we should just delete it?
113 newpath = base + '-corrupt' + ext
113 newpath = base + '-corrupt' + ext
114 os.rename(self.hist_file, newpath)
114 os.rename(self.hist_file, newpath)
115 self.log.error("History file was moved to %s and a new file created.", newpath)
115 self.log.error("History file was moved to %s and a new file created.", newpath)
116 self.init_db()
116 self.init_db()
117 return []
117 return []
118 else:
118 else:
119 # Failed with :memory:, something serious is wrong
119 # Failed with :memory:, something serious is wrong
120 raise
120 raise
121
121
122 class HistoryAccessorBase(LoggingConfigurable):
122 class HistoryAccessorBase(LoggingConfigurable):
123 """An abstract class for History Accessors """
123 """An abstract class for History Accessors """
124
124
125 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
125 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
126 raise NotImplementedError
126 raise NotImplementedError
127
127
128 def search(self, pattern="*", raw=True, search_raw=True,
128 def search(self, pattern="*", raw=True, search_raw=True,
129 output=False, n=None, unique=False):
129 output=False, n=None, unique=False):
130 raise NotImplementedError
130 raise NotImplementedError
131
131
132 def get_range(self, session, start=1, stop=None, raw=True,output=False):
132 def get_range(self, session, start=1, stop=None, raw=True,output=False):
133 raise NotImplementedError
133 raise NotImplementedError
134
134
135 def get_range_by_str(self, rangestr, raw=True, output=False):
135 def get_range_by_str(self, rangestr, raw=True, output=False):
136 raise NotImplementedError
136 raise NotImplementedError
137
137
138
138
139 class HistoryAccessor(HistoryAccessorBase):
139 class HistoryAccessor(HistoryAccessorBase):
140 """Access the history database without adding to it.
140 """Access the history database without adding to it.
141
141
142 This is intended for use by standalone history tools. IPython shells use
142 This is intended for use by standalone history tools. IPython shells use
143 HistoryManager, below, which is a subclass of this."""
143 HistoryManager, below, which is a subclass of this."""
144
144
145 # counter for init_db retries, so we don't keep trying over and over
145 # counter for init_db retries, so we don't keep trying over and over
146 _corrupt_db_counter = 0
146 _corrupt_db_counter = 0
147 # after two failures, fallback on :memory:
147 # after two failures, fallback on :memory:
148 _corrupt_db_limit = 2
148 _corrupt_db_limit = 2
149
149
150 # String holding the path to the history file
150 # String holding the path to the history file
151 hist_file = Unicode(
151 hist_file = Unicode(
152 help="""Path to file to use for SQLite history database.
152 help="""Path to file to use for SQLite history database.
153
153
154 By default, IPython will put the history database in the IPython
154 By default, IPython will put the history database in the IPython
155 profile directory. If you would rather share one history among
155 profile directory. If you would rather share one history among
156 profiles, you can set this value in each, so that they are consistent.
156 profiles, you can set this value in each, so that they are consistent.
157
157
158 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
158 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
159 mounts. If you see IPython hanging, try setting this to something on a
159 mounts. If you see IPython hanging, try setting this to something on a
160 local disk, e.g::
160 local disk, e.g::
161
161
162 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
162 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
163
163
164 you can also use the specific value `:memory:` (including the colon
164 you can also use the specific value `:memory:` (including the colon
165 at both end but not the back ticks), to avoid creating an history file.
165 at both end but not the back ticks), to avoid creating an history file.
166
166
167 """).tag(config=True)
167 """).tag(config=True)
168
168
169 enabled = Bool(True,
169 enabled = Bool(True,
170 help="""enable the SQLite history
170 help="""enable the SQLite history
171
171
172 set enabled=False to disable the SQLite history,
172 set enabled=False to disable the SQLite history,
173 in which case there will be no stored history, no SQLite connection,
173 in which case there will be no stored history, no SQLite connection,
174 and no background saving thread. This may be necessary in some
174 and no background saving thread. This may be necessary in some
175 threaded environments where IPython is embedded.
175 threaded environments where IPython is embedded.
176 """
176 """
177 ).tag(config=True)
177 ).tag(config=True)
178
178
179 connection_options = Dict(
179 connection_options = Dict(
180 help="""Options for configuring the SQLite connection
180 help="""Options for configuring the SQLite connection
181
181
182 These options are passed as keyword args to sqlite3.connect
182 These options are passed as keyword args to sqlite3.connect
183 when establishing database conenctions.
183 when establishing database conenctions.
184 """
184 """
185 ).tag(config=True)
185 ).tag(config=True)
186
186
187 # The SQLite database
187 # The SQLite database
188 db = Any()
188 db = Any()
189 @observe('db')
189 @observe('db')
190 def _db_changed(self, change):
190 def _db_changed(self, change):
191 """validate the db, since it can be an Instance of two different types"""
191 """validate the db, since it can be an Instance of two different types"""
192 new = change['new']
192 new = change['new']
193 connection_types = (DummyDB,)
193 connection_types = (DummyDB,)
194 if sqlite3 is not None:
194 if sqlite3 is not None:
195 connection_types = (DummyDB, sqlite3.Connection)
195 connection_types = (DummyDB, sqlite3.Connection)
196 if not isinstance(new, connection_types):
196 if not isinstance(new, connection_types):
197 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
197 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
198 (self.__class__.__name__, new)
198 (self.__class__.__name__, new)
199 raise TraitError(msg)
199 raise TraitError(msg)
200
200
201 def __init__(self, profile='default', hist_file=u'', **traits):
201 def __init__(self, profile='default', hist_file=u'', **traits):
202 """Create a new history accessor.
202 """Create a new history accessor.
203
203
204 Parameters
204 Parameters
205 ----------
205 ----------
206 profile : str
206 profile : str
207 The name of the profile from which to open history.
207 The name of the profile from which to open history.
208 hist_file : str
208 hist_file : str
209 Path to an SQLite history database stored by IPython. If specified,
209 Path to an SQLite history database stored by IPython. If specified,
210 hist_file overrides profile.
210 hist_file overrides profile.
211 config : :class:`~traitlets.config.loader.Config`
211 config : :class:`~traitlets.config.loader.Config`
212 Config object. hist_file can also be set through this.
212 Config object. hist_file can also be set through this.
213 """
213 """
214 # We need a pointer back to the shell for various tasks.
214 # We need a pointer back to the shell for various tasks.
215 super(HistoryAccessor, self).__init__(**traits)
215 super(HistoryAccessor, self).__init__(**traits)
216 # defer setting hist_file from kwarg until after init,
216 # defer setting hist_file from kwarg until after init,
217 # otherwise the default kwarg value would clobber any value
217 # otherwise the default kwarg value would clobber any value
218 # set by config
218 # set by config
219 if hist_file:
219 if hist_file:
220 self.hist_file = hist_file
220 self.hist_file = hist_file
221
221
222 if self.hist_file == u'':
222 if self.hist_file == u'':
223 # No one has set the hist_file, yet.
223 # No one has set the hist_file, yet.
224 self.hist_file = self._get_hist_file_name(profile)
224 self.hist_file = self._get_hist_file_name(profile)
225
225
226 if sqlite3 is None and self.enabled:
226 if sqlite3 is None and self.enabled:
227 warn("IPython History requires SQLite, your history will not be saved")
227 warn("IPython History requires SQLite, your history will not be saved")
228 self.enabled = False
228 self.enabled = False
229
229
230 self.init_db()
230 self.init_db()
231
231
232 def _get_hist_file_name(self, profile='default'):
232 def _get_hist_file_name(self, profile='default'):
233 """Find the history file for the given profile name.
233 """Find the history file for the given profile name.
234
234
235 This is overridden by the HistoryManager subclass, to use the shell's
235 This is overridden by the HistoryManager subclass, to use the shell's
236 active profile.
236 active profile.
237
237
238 Parameters
238 Parameters
239 ----------
239 ----------
240 profile : str
240 profile : str
241 The name of a profile which has a history file.
241 The name of a profile which has a history file.
242 """
242 """
243 return os.path.join(locate_profile(profile), 'history.sqlite')
243 return os.path.join(locate_profile(profile), 'history.sqlite')
244
244
245 @catch_corrupt_db
245 @catch_corrupt_db
246 def init_db(self):
246 def init_db(self):
247 """Connect to the database, and create tables if necessary."""
247 """Connect to the database, and create tables if necessary."""
248 if not self.enabled:
248 if not self.enabled:
249 self.db = DummyDB()
249 self.db = DummyDB()
250 return
250 return
251
251
252 # use detect_types so that timestamps return datetime objects
252 # use detect_types so that timestamps return datetime objects
253 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
253 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
254 kwargs.update(self.connection_options)
254 kwargs.update(self.connection_options)
255 self.db = sqlite3.connect(self.hist_file, **kwargs)
255 self.db = sqlite3.connect(self.hist_file, **kwargs)
256 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
256 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
257 primary key autoincrement, start timestamp,
257 primary key autoincrement, start timestamp,
258 end timestamp, num_cmds integer, remark text)""")
258 end timestamp, num_cmds integer, remark text)""")
259 self.db.execute("""CREATE TABLE IF NOT EXISTS history
259 self.db.execute("""CREATE TABLE IF NOT EXISTS history
260 (session integer, line integer, source text, source_raw text,
260 (session integer, line integer, source text, source_raw text,
261 PRIMARY KEY (session, line))""")
261 PRIMARY KEY (session, line))""")
262 # Output history is optional, but ensure the table's there so it can be
262 # Output history is optional, but ensure the table's there so it can be
263 # enabled later.
263 # enabled later.
264 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
264 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
265 (session integer, line integer, output text,
265 (session integer, line integer, output text,
266 PRIMARY KEY (session, line))""")
266 PRIMARY KEY (session, line))""")
267 self.db.commit()
267 self.db.commit()
268 # success! reset corrupt db count
268 # success! reset corrupt db count
269 self._corrupt_db_counter = 0
269 self._corrupt_db_counter = 0
270
270
271 def writeout_cache(self):
271 def writeout_cache(self):
272 """Overridden by HistoryManager to dump the cache before certain
272 """Overridden by HistoryManager to dump the cache before certain
273 database lookups."""
273 database lookups."""
274 pass
274 pass
275
275
276 ## -------------------------------
276 ## -------------------------------
277 ## Methods for retrieving history:
277 ## Methods for retrieving history:
278 ## -------------------------------
278 ## -------------------------------
279 def _run_sql(self, sql, params, raw=True, output=False):
279 def _run_sql(self, sql, params, raw=True, output=False):
280 """Prepares and runs an SQL query for the history database.
280 """Prepares and runs an SQL query for the history database.
281
281
282 Parameters
282 Parameters
283 ----------
283 ----------
284 sql : str
284 sql : str
285 Any filtering expressions to go after SELECT ... FROM ...
285 Any filtering expressions to go after SELECT ... FROM ...
286 params : tuple
286 params : tuple
287 Parameters passed to the SQL query (to replace "?")
287 Parameters passed to the SQL query (to replace "?")
288 raw, output : bool
288 raw, output : bool
289 See :meth:`get_range`
289 See :meth:`get_range`
290
290
291 Returns
291 Returns
292 -------
292 -------
293 Tuples as :meth:`get_range`
293 Tuples as :meth:`get_range`
294 """
294 """
295 toget = 'source_raw' if raw else 'source'
295 toget = 'source_raw' if raw else 'source'
296 sqlfrom = "history"
296 sqlfrom = "history"
297 if output:
297 if output:
298 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
298 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
299 toget = "history.%s, output_history.output" % toget
299 toget = "history.%s, output_history.output" % toget
300 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
300 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
301 (toget, sqlfrom) + sql, params)
301 (toget, sqlfrom) + sql, params)
302 if output: # Regroup into 3-tuples, and parse JSON
302 if output: # Regroup into 3-tuples, and parse JSON
303 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
303 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
304 return cur
304 return cur
305
305
306 @needs_sqlite
306 @needs_sqlite
307 @catch_corrupt_db
307 @catch_corrupt_db
308 def get_session_info(self, session):
308 def get_session_info(self, session):
309 """Get info about a session.
309 """Get info about a session.
310
310
311 Parameters
311 Parameters
312 ----------
312 ----------
313
313
314 session : int
314 session : int
315 Session number to retrieve.
315 Session number to retrieve.
316
316
317 Returns
317 Returns
318 -------
318 -------
319
319
320 session_id : int
320 session_id : int
321 Session ID number
321 Session ID number
322 start : datetime
322 start : datetime
323 Timestamp for the start of the session.
323 Timestamp for the start of the session.
324 end : datetime
324 end : datetime
325 Timestamp for the end of the session, or None if IPython crashed.
325 Timestamp for the end of the session, or None if IPython crashed.
326 num_cmds : int
326 num_cmds : int
327 Number of commands run, or None if IPython crashed.
327 Number of commands run, or None if IPython crashed.
328 remark : unicode
328 remark : unicode
329 A manually set description.
329 A manually set description.
330 """
330 """
331 query = "SELECT * from sessions where session == ?"
331 query = "SELECT * from sessions where session == ?"
332 return self.db.execute(query, (session,)).fetchone()
332 return self.db.execute(query, (session,)).fetchone()
333
333
334 @catch_corrupt_db
334 @catch_corrupt_db
335 def get_last_session_id(self):
335 def get_last_session_id(self):
336 """Get the last session ID currently in the database.
336 """Get the last session ID currently in the database.
337
337
338 Within IPython, this should be the same as the value stored in
338 Within IPython, this should be the same as the value stored in
339 :attr:`HistoryManager.session_number`.
339 :attr:`HistoryManager.session_number`.
340 """
340 """
341 for record in self.get_tail(n=1, include_latest=True):
341 for record in self.get_tail(n=1, include_latest=True):
342 return record[0]
342 return record[0]
343
343
344 @catch_corrupt_db
344 @catch_corrupt_db
345 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
345 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
346 """Get the last n lines from the history database.
346 """Get the last n lines from the history database.
347
347
348 Parameters
348 Parameters
349 ----------
349 ----------
350 n : int
350 n : int
351 The number of lines to get
351 The number of lines to get
352 raw, output : bool
352 raw, output : bool
353 See :meth:`get_range`
353 See :meth:`get_range`
354 include_latest : bool
354 include_latest : bool
355 If False (default), n+1 lines are fetched, and the latest one
355 If False (default), n+1 lines are fetched, and the latest one
356 is discarded. This is intended to be used where the function
356 is discarded. This is intended to be used where the function
357 is called by a user command, which it should not return.
357 is called by a user command, which it should not return.
358
358
359 Returns
359 Returns
360 -------
360 -------
361 Tuples as :meth:`get_range`
361 Tuples as :meth:`get_range`
362 """
362 """
363 self.writeout_cache()
363 self.writeout_cache()
364 if not include_latest:
364 if not include_latest:
365 n += 1
365 n += 1
366 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
366 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
367 (n,), raw=raw, output=output)
367 (n,), raw=raw, output=output)
368 if not include_latest:
368 if not include_latest:
369 return reversed(list(cur)[1:])
369 return reversed(list(cur)[1:])
370 return reversed(list(cur))
370 return reversed(list(cur))
371
371
372 @catch_corrupt_db
372 @catch_corrupt_db
373 def search(self, pattern="*", raw=True, search_raw=True,
373 def search(self, pattern="*", raw=True, search_raw=True,
374 output=False, n=None, unique=False):
374 output=False, n=None, unique=False):
375 """Search the database using unix glob-style matching (wildcards
375 """Search the database using unix glob-style matching (wildcards
376 * and ?).
376 * and ?).
377
377
378 Parameters
378 Parameters
379 ----------
379 ----------
380 pattern : str
380 pattern : str
381 The wildcarded pattern to match when searching
381 The wildcarded pattern to match when searching
382 search_raw : bool
382 search_raw : bool
383 If True, search the raw input, otherwise, the parsed input
383 If True, search the raw input, otherwise, the parsed input
384 raw, output : bool
384 raw, output : bool
385 See :meth:`get_range`
385 See :meth:`get_range`
386 n : None or int
386 n : None or int
387 If an integer is given, it defines the limit of
387 If an integer is given, it defines the limit of
388 returned entries.
388 returned entries.
389 unique : bool
389 unique : bool
390 When it is true, return only unique entries.
390 When it is true, return only unique entries.
391
391
392 Returns
392 Returns
393 -------
393 -------
394 Tuples as :meth:`get_range`
394 Tuples as :meth:`get_range`
395 """
395 """
396 tosearch = "source_raw" if search_raw else "source"
396 tosearch = "source_raw" if search_raw else "source"
397 if output:
397 if output:
398 tosearch = "history." + tosearch
398 tosearch = "history." + tosearch
399 self.writeout_cache()
399 self.writeout_cache()
400 sqlform = "WHERE %s GLOB ?" % tosearch
400 sqlform = "WHERE %s GLOB ?" % tosearch
401 params = (pattern,)
401 params = (pattern,)
402 if unique:
402 if unique:
403 sqlform += ' GROUP BY {0}'.format(tosearch)
403 sqlform += ' GROUP BY {0}'.format(tosearch)
404 if n is not None:
404 if n is not None:
405 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
405 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
406 params += (n,)
406 params += (n,)
407 elif unique:
407 elif unique:
408 sqlform += " ORDER BY session, line"
408 sqlform += " ORDER BY session, line"
409 cur = self._run_sql(sqlform, params, raw=raw, output=output)
409 cur = self._run_sql(sqlform, params, raw=raw, output=output)
410 if n is not None:
410 if n is not None:
411 return reversed(list(cur))
411 return reversed(list(cur))
412 return cur
412 return cur
413
413
414 @catch_corrupt_db
414 @catch_corrupt_db
415 def get_range(self, session, start=1, stop=None, raw=True,output=False):
415 def get_range(self, session, start=1, stop=None, raw=True,output=False):
416 """Retrieve input by session.
416 """Retrieve input by session.
417
417
418 Parameters
418 Parameters
419 ----------
419 ----------
420 session : int
420 session : int
421 Session number to retrieve.
421 Session number to retrieve.
422 start : int
422 start : int
423 First line to retrieve.
423 First line to retrieve.
424 stop : int
424 stop : int
425 End of line range (excluded from output itself). If None, retrieve
425 End of line range (excluded from output itself). If None, retrieve
426 to the end of the session.
426 to the end of the session.
427 raw : bool
427 raw : bool
428 If True, return untranslated input
428 If True, return untranslated input
429 output : bool
429 output : bool
430 If True, attempt to include output. This will be 'real' Python
430 If True, attempt to include output. This will be 'real' Python
431 objects for the current session, or text reprs from previous
431 objects for the current session, or text reprs from previous
432 sessions if db_log_output was enabled at the time. Where no output
432 sessions if db_log_output was enabled at the time. Where no output
433 is found, None is used.
433 is found, None is used.
434
434
435 Returns
435 Returns
436 -------
436 -------
437 entries
437 entries
438 An iterator over the desired lines. Each line is a 3-tuple, either
438 An iterator over the desired lines. Each line is a 3-tuple, either
439 (session, line, input) if output is False, or
439 (session, line, input) if output is False, or
440 (session, line, (input, output)) if output is True.
440 (session, line, (input, output)) if output is True.
441 """
441 """
442 if stop:
442 if stop:
443 lineclause = "line >= ? AND line < ?"
443 lineclause = "line >= ? AND line < ?"
444 params = (session, start, stop)
444 params = (session, start, stop)
445 else:
445 else:
446 lineclause = "line>=?"
446 lineclause = "line>=?"
447 params = (session, start)
447 params = (session, start)
448
448
449 return self._run_sql("WHERE session==? AND %s" % lineclause,
449 return self._run_sql("WHERE session==? AND %s" % lineclause,
450 params, raw=raw, output=output)
450 params, raw=raw, output=output)
451
451
452 def get_range_by_str(self, rangestr, raw=True, output=False):
452 def get_range_by_str(self, rangestr, raw=True, output=False):
453 """Get lines of history from a string of ranges, as used by magic
453 """Get lines of history from a string of ranges, as used by magic
454 commands %hist, %save, %macro, etc.
454 commands %hist, %save, %macro, etc.
455
455
456 Parameters
456 Parameters
457 ----------
457 ----------
458 rangestr : str
458 rangestr : str
459 A string specifying ranges, e.g. "5 ~2/1-4". See
459 A string specifying ranges, e.g. "5 ~2/1-4". See
460 :func:`magic_history` for full details.
460 :func:`magic_history` for full details.
461 raw, output : bool
461 raw, output : bool
462 As :meth:`get_range`
462 As :meth:`get_range`
463
463
464 Returns
464 Returns
465 -------
465 -------
466 Tuples as :meth:`get_range`
466 Tuples as :meth:`get_range`
467 """
467 """
468 for sess, s, e in extract_hist_ranges(rangestr):
468 for sess, s, e in extract_hist_ranges(rangestr):
469 for line in self.get_range(sess, s, e, raw=raw, output=output):
469 for line in self.get_range(sess, s, e, raw=raw, output=output):
470 yield line
470 yield line
471
471
472
472
473 class HistoryManager(HistoryAccessor):
473 class HistoryManager(HistoryAccessor):
474 """A class to organize all history-related functionality in one place.
474 """A class to organize all history-related functionality in one place.
475 """
475 """
476 # Public interface
476 # Public interface
477
477
478 # An instance of the IPython shell we are attached to
478 # An instance of the IPython shell we are attached to
479 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
479 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
480 allow_none=True)
480 allow_none=True)
481 # Lists to hold processed and raw history. These start with a blank entry
481 # Lists to hold processed and raw history. These start with a blank entry
482 # so that we can index them starting from 1
482 # so that we can index them starting from 1
483 input_hist_parsed = List([""])
483 input_hist_parsed = List([""])
484 input_hist_raw = List([""])
484 input_hist_raw = List([""])
485 # A list of directories visited during session
485 # A list of directories visited during session
486 dir_hist = List()
486 dir_hist = List()
487 @default('dir_hist')
487 @default('dir_hist')
488 def _dir_hist_default(self):
488 def _dir_hist_default(self):
489 try:
489 try:
490 return [os.getcwd()]
490 return [os.getcwd()]
491 except OSError:
491 except OSError:
492 return []
492 return []
493
493
494 # A dict of output history, keyed with ints from the shell's
494 # A dict of output history, keyed with ints from the shell's
495 # execution count.
495 # execution count.
496 output_hist = Dict()
496 output_hist = Dict()
497 # The text/plain repr of outputs.
497 # The text/plain repr of outputs.
498 output_hist_reprs = Dict()
498 output_hist_reprs = Dict()
499
499
500 # The number of the current session in the history database
500 # The number of the current session in the history database
501 session_number = Integer()
501 session_number = Integer()
502
502
503 db_log_output = Bool(False,
503 db_log_output = Bool(False,
504 help="Should the history database include output? (default: no)"
504 help="Should the history database include output? (default: no)"
505 ).tag(config=True)
505 ).tag(config=True)
506 db_cache_size = Integer(0,
506 db_cache_size = Integer(0,
507 help="Write to database every x commands (higher values save disk access & power).\n"
507 help="Write to database every x commands (higher values save disk access & power).\n"
508 "Values of 1 or less effectively disable caching."
508 "Values of 1 or less effectively disable caching."
509 ).tag(config=True)
509 ).tag(config=True)
510 # The input and output caches
510 # The input and output caches
511 db_input_cache = List()
511 db_input_cache = List()
512 db_output_cache = List()
512 db_output_cache = List()
513
513
514 # History saving in separate thread
514 # History saving in separate thread
515 save_thread = Instance('IPython.core.history.HistorySavingThread',
515 save_thread = Instance('IPython.core.history.HistorySavingThread',
516 allow_none=True)
516 allow_none=True)
517 try: # Event is a function returning an instance of _Event...
517 save_flag = Instance(threading.Event, allow_none=True)
518 save_flag = Instance(threading._Event, allow_none=True)
519 except AttributeError: # ...until Python 3.3, when it's a class.
520 save_flag = Instance(threading.Event, allow_none=True)
521
518
522 # Private interface
519 # Private interface
523 # Variables used to store the three last inputs from the user. On each new
520 # Variables used to store the three last inputs from the user. On each new
524 # history update, we populate the user's namespace with these, shifted as
521 # history update, we populate the user's namespace with these, shifted as
525 # necessary.
522 # necessary.
526 _i00 = Unicode(u'')
523 _i00 = Unicode(u'')
527 _i = Unicode(u'')
524 _i = Unicode(u'')
528 _ii = Unicode(u'')
525 _ii = Unicode(u'')
529 _iii = Unicode(u'')
526 _iii = Unicode(u'')
530
527
531 # A regex matching all forms of the exit command, so that we don't store
528 # A regex matching all forms of the exit command, so that we don't store
532 # them in the history (it's annoying to rewind the first entry and land on
529 # them in the history (it's annoying to rewind the first entry and land on
533 # an exit call).
530 # an exit call).
534 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
531 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
535
532
536 def __init__(self, shell=None, config=None, **traits):
533 def __init__(self, shell=None, config=None, **traits):
537 """Create a new history manager associated with a shell instance.
534 """Create a new history manager associated with a shell instance.
538 """
535 """
539 # We need a pointer back to the shell for various tasks.
536 # We need a pointer back to the shell for various tasks.
540 super(HistoryManager, self).__init__(shell=shell, config=config,
537 super(HistoryManager, self).__init__(shell=shell, config=config,
541 **traits)
538 **traits)
542 self.save_flag = threading.Event()
539 self.save_flag = threading.Event()
543 self.db_input_cache_lock = threading.Lock()
540 self.db_input_cache_lock = threading.Lock()
544 self.db_output_cache_lock = threading.Lock()
541 self.db_output_cache_lock = threading.Lock()
545
542
546 try:
543 try:
547 self.new_session()
544 self.new_session()
548 except OperationalError:
545 except OperationalError:
549 self.log.error("Failed to create history session in %s. History will not be saved.",
546 self.log.error("Failed to create history session in %s. History will not be saved.",
550 self.hist_file, exc_info=True)
547 self.hist_file, exc_info=True)
551 self.hist_file = ':memory:'
548 self.hist_file = ':memory:'
552
549
553 if self.enabled and self.hist_file != ':memory:':
550 if self.enabled and self.hist_file != ':memory:':
554 self.save_thread = HistorySavingThread(self)
551 self.save_thread = HistorySavingThread(self)
555 self.save_thread.start()
552 self.save_thread.start()
556
553
557 def _get_hist_file_name(self, profile=None):
554 def _get_hist_file_name(self, profile=None):
558 """Get default history file name based on the Shell's profile.
555 """Get default history file name based on the Shell's profile.
559
556
560 The profile parameter is ignored, but must exist for compatibility with
557 The profile parameter is ignored, but must exist for compatibility with
561 the parent class."""
558 the parent class."""
562 profile_dir = self.shell.profile_dir.location
559 profile_dir = self.shell.profile_dir.location
563 return os.path.join(profile_dir, 'history.sqlite')
560 return os.path.join(profile_dir, 'history.sqlite')
564
561
565 @needs_sqlite
562 @needs_sqlite
566 def new_session(self, conn=None):
563 def new_session(self, conn=None):
567 """Get a new session number."""
564 """Get a new session number."""
568 if conn is None:
565 if conn is None:
569 conn = self.db
566 conn = self.db
570
567
571 with conn:
568 with conn:
572 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
569 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
573 NULL, "") """, (datetime.datetime.now(),))
570 NULL, "") """, (datetime.datetime.now(),))
574 self.session_number = cur.lastrowid
571 self.session_number = cur.lastrowid
575
572
576 def end_session(self):
573 def end_session(self):
577 """Close the database session, filling in the end time and line count."""
574 """Close the database session, filling in the end time and line count."""
578 self.writeout_cache()
575 self.writeout_cache()
579 with self.db:
576 with self.db:
580 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
577 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
581 session==?""", (datetime.datetime.now(),
578 session==?""", (datetime.datetime.now(),
582 len(self.input_hist_parsed)-1, self.session_number))
579 len(self.input_hist_parsed)-1, self.session_number))
583 self.session_number = 0
580 self.session_number = 0
584
581
585 def name_session(self, name):
582 def name_session(self, name):
586 """Give the current session a name in the history database."""
583 """Give the current session a name in the history database."""
587 with self.db:
584 with self.db:
588 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
585 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
589 (name, self.session_number))
586 (name, self.session_number))
590
587
591 def reset(self, new_session=True):
588 def reset(self, new_session=True):
592 """Clear the session history, releasing all object references, and
589 """Clear the session history, releasing all object references, and
593 optionally open a new session."""
590 optionally open a new session."""
594 self.output_hist.clear()
591 self.output_hist.clear()
595 # The directory history can't be completely empty
592 # The directory history can't be completely empty
596 self.dir_hist[:] = [os.getcwd()]
593 self.dir_hist[:] = [os.getcwd()]
597
594
598 if new_session:
595 if new_session:
599 if self.session_number:
596 if self.session_number:
600 self.end_session()
597 self.end_session()
601 self.input_hist_parsed[:] = [""]
598 self.input_hist_parsed[:] = [""]
602 self.input_hist_raw[:] = [""]
599 self.input_hist_raw[:] = [""]
603 self.new_session()
600 self.new_session()
604
601
605 # ------------------------------
602 # ------------------------------
606 # Methods for retrieving history
603 # Methods for retrieving history
607 # ------------------------------
604 # ------------------------------
608 def get_session_info(self, session=0):
605 def get_session_info(self, session=0):
609 """Get info about a session.
606 """Get info about a session.
610
607
611 Parameters
608 Parameters
612 ----------
609 ----------
613
610
614 session : int
611 session : int
615 Session number to retrieve. The current session is 0, and negative
612 Session number to retrieve. The current session is 0, and negative
616 numbers count back from current session, so -1 is the previous session.
613 numbers count back from current session, so -1 is the previous session.
617
614
618 Returns
615 Returns
619 -------
616 -------
620
617
621 session_id : int
618 session_id : int
622 Session ID number
619 Session ID number
623 start : datetime
620 start : datetime
624 Timestamp for the start of the session.
621 Timestamp for the start of the session.
625 end : datetime
622 end : datetime
626 Timestamp for the end of the session, or None if IPython crashed.
623 Timestamp for the end of the session, or None if IPython crashed.
627 num_cmds : int
624 num_cmds : int
628 Number of commands run, or None if IPython crashed.
625 Number of commands run, or None if IPython crashed.
629 remark : unicode
626 remark : unicode
630 A manually set description.
627 A manually set description.
631 """
628 """
632 if session <= 0:
629 if session <= 0:
633 session += self.session_number
630 session += self.session_number
634
631
635 return super(HistoryManager, self).get_session_info(session=session)
632 return super(HistoryManager, self).get_session_info(session=session)
636
633
637 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
634 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
638 """Get input and output history from the current session. Called by
635 """Get input and output history from the current session. Called by
639 get_range, and takes similar parameters."""
636 get_range, and takes similar parameters."""
640 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
637 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
641
638
642 n = len(input_hist)
639 n = len(input_hist)
643 if start < 0:
640 if start < 0:
644 start += n
641 start += n
645 if not stop or (stop > n):
642 if not stop or (stop > n):
646 stop = n
643 stop = n
647 elif stop < 0:
644 elif stop < 0:
648 stop += n
645 stop += n
649
646
650 for i in range(start, stop):
647 for i in range(start, stop):
651 if output:
648 if output:
652 line = (input_hist[i], self.output_hist_reprs.get(i))
649 line = (input_hist[i], self.output_hist_reprs.get(i))
653 else:
650 else:
654 line = input_hist[i]
651 line = input_hist[i]
655 yield (0, i, line)
652 yield (0, i, line)
656
653
657 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
654 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
658 """Retrieve input by session.
655 """Retrieve input by session.
659
656
660 Parameters
657 Parameters
661 ----------
658 ----------
662 session : int
659 session : int
663 Session number to retrieve. The current session is 0, and negative
660 Session number to retrieve. The current session is 0, and negative
664 numbers count back from current session, so -1 is previous session.
661 numbers count back from current session, so -1 is previous session.
665 start : int
662 start : int
666 First line to retrieve.
663 First line to retrieve.
667 stop : int
664 stop : int
668 End of line range (excluded from output itself). If None, retrieve
665 End of line range (excluded from output itself). If None, retrieve
669 to the end of the session.
666 to the end of the session.
670 raw : bool
667 raw : bool
671 If True, return untranslated input
668 If True, return untranslated input
672 output : bool
669 output : bool
673 If True, attempt to include output. This will be 'real' Python
670 If True, attempt to include output. This will be 'real' Python
674 objects for the current session, or text reprs from previous
671 objects for the current session, or text reprs from previous
675 sessions if db_log_output was enabled at the time. Where no output
672 sessions if db_log_output was enabled at the time. Where no output
676 is found, None is used.
673 is found, None is used.
677
674
678 Returns
675 Returns
679 -------
676 -------
680 entries
677 entries
681 An iterator over the desired lines. Each line is a 3-tuple, either
678 An iterator over the desired lines. Each line is a 3-tuple, either
682 (session, line, input) if output is False, or
679 (session, line, input) if output is False, or
683 (session, line, (input, output)) if output is True.
680 (session, line, (input, output)) if output is True.
684 """
681 """
685 if session <= 0:
682 if session <= 0:
686 session += self.session_number
683 session += self.session_number
687 if session==self.session_number: # Current session
684 if session==self.session_number: # Current session
688 return self._get_range_session(start, stop, raw, output)
685 return self._get_range_session(start, stop, raw, output)
689 return super(HistoryManager, self).get_range(session, start, stop, raw,
686 return super(HistoryManager, self).get_range(session, start, stop, raw,
690 output)
687 output)
691
688
692 ## ----------------------------
689 ## ----------------------------
693 ## Methods for storing history:
690 ## Methods for storing history:
694 ## ----------------------------
691 ## ----------------------------
695 def store_inputs(self, line_num, source, source_raw=None):
692 def store_inputs(self, line_num, source, source_raw=None):
696 """Store source and raw input in history and create input cache
693 """Store source and raw input in history and create input cache
697 variables ``_i*``.
694 variables ``_i*``.
698
695
699 Parameters
696 Parameters
700 ----------
697 ----------
701 line_num : int
698 line_num : int
702 The prompt number of this input.
699 The prompt number of this input.
703
700
704 source : str
701 source : str
705 Python input.
702 Python input.
706
703
707 source_raw : str, optional
704 source_raw : str, optional
708 If given, this is the raw input without any IPython transformations
705 If given, this is the raw input without any IPython transformations
709 applied to it. If not given, ``source`` is used.
706 applied to it. If not given, ``source`` is used.
710 """
707 """
711 if source_raw is None:
708 if source_raw is None:
712 source_raw = source
709 source_raw = source
713 source = source.rstrip('\n')
710 source = source.rstrip('\n')
714 source_raw = source_raw.rstrip('\n')
711 source_raw = source_raw.rstrip('\n')
715
712
716 # do not store exit/quit commands
713 # do not store exit/quit commands
717 if self._exit_re.match(source_raw.strip()):
714 if self._exit_re.match(source_raw.strip()):
718 return
715 return
719
716
720 self.input_hist_parsed.append(source)
717 self.input_hist_parsed.append(source)
721 self.input_hist_raw.append(source_raw)
718 self.input_hist_raw.append(source_raw)
722
719
723 with self.db_input_cache_lock:
720 with self.db_input_cache_lock:
724 self.db_input_cache.append((line_num, source, source_raw))
721 self.db_input_cache.append((line_num, source, source_raw))
725 # Trigger to flush cache and write to DB.
722 # Trigger to flush cache and write to DB.
726 if len(self.db_input_cache) >= self.db_cache_size:
723 if len(self.db_input_cache) >= self.db_cache_size:
727 self.save_flag.set()
724 self.save_flag.set()
728
725
729 # update the auto _i variables
726 # update the auto _i variables
730 self._iii = self._ii
727 self._iii = self._ii
731 self._ii = self._i
728 self._ii = self._i
732 self._i = self._i00
729 self._i = self._i00
733 self._i00 = source_raw
730 self._i00 = source_raw
734
731
735 # hackish access to user namespace to create _i1,_i2... dynamically
732 # hackish access to user namespace to create _i1,_i2... dynamically
736 new_i = '_i%s' % line_num
733 new_i = '_i%s' % line_num
737 to_main = {'_i': self._i,
734 to_main = {'_i': self._i,
738 '_ii': self._ii,
735 '_ii': self._ii,
739 '_iii': self._iii,
736 '_iii': self._iii,
740 new_i : self._i00 }
737 new_i : self._i00 }
741
738
742 if self.shell is not None:
739 if self.shell is not None:
743 self.shell.push(to_main, interactive=False)
740 self.shell.push(to_main, interactive=False)
744
741
745 def store_output(self, line_num):
742 def store_output(self, line_num):
746 """If database output logging is enabled, this saves all the
743 """If database output logging is enabled, this saves all the
747 outputs from the indicated prompt number to the database. It's
744 outputs from the indicated prompt number to the database. It's
748 called by run_cell after code has been executed.
745 called by run_cell after code has been executed.
749
746
750 Parameters
747 Parameters
751 ----------
748 ----------
752 line_num : int
749 line_num : int
753 The line number from which to save outputs
750 The line number from which to save outputs
754 """
751 """
755 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
752 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
756 return
753 return
757 output = self.output_hist_reprs[line_num]
754 output = self.output_hist_reprs[line_num]
758
755
759 with self.db_output_cache_lock:
756 with self.db_output_cache_lock:
760 self.db_output_cache.append((line_num, output))
757 self.db_output_cache.append((line_num, output))
761 if self.db_cache_size <= 1:
758 if self.db_cache_size <= 1:
762 self.save_flag.set()
759 self.save_flag.set()
763
760
764 def _writeout_input_cache(self, conn):
761 def _writeout_input_cache(self, conn):
765 with conn:
762 with conn:
766 for line in self.db_input_cache:
763 for line in self.db_input_cache:
767 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
764 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
768 (self.session_number,)+line)
765 (self.session_number,)+line)
769
766
770 def _writeout_output_cache(self, conn):
767 def _writeout_output_cache(self, conn):
771 with conn:
768 with conn:
772 for line in self.db_output_cache:
769 for line in self.db_output_cache:
773 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
770 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
774 (self.session_number,)+line)
771 (self.session_number,)+line)
775
772
776 @needs_sqlite
773 @needs_sqlite
777 def writeout_cache(self, conn=None):
774 def writeout_cache(self, conn=None):
778 """Write any entries in the cache to the database."""
775 """Write any entries in the cache to the database."""
779 if conn is None:
776 if conn is None:
780 conn = self.db
777 conn = self.db
781
778
782 with self.db_input_cache_lock:
779 with self.db_input_cache_lock:
783 try:
780 try:
784 self._writeout_input_cache(conn)
781 self._writeout_input_cache(conn)
785 except sqlite3.IntegrityError:
782 except sqlite3.IntegrityError:
786 self.new_session(conn)
783 self.new_session(conn)
787 print("ERROR! Session/line number was not unique in",
784 print("ERROR! Session/line number was not unique in",
788 "database. History logging moved to new session",
785 "database. History logging moved to new session",
789 self.session_number)
786 self.session_number)
790 try:
787 try:
791 # Try writing to the new session. If this fails, don't
788 # Try writing to the new session. If this fails, don't
792 # recurse
789 # recurse
793 self._writeout_input_cache(conn)
790 self._writeout_input_cache(conn)
794 except sqlite3.IntegrityError:
791 except sqlite3.IntegrityError:
795 pass
792 pass
796 finally:
793 finally:
797 self.db_input_cache = []
794 self.db_input_cache = []
798
795
799 with self.db_output_cache_lock:
796 with self.db_output_cache_lock:
800 try:
797 try:
801 self._writeout_output_cache(conn)
798 self._writeout_output_cache(conn)
802 except sqlite3.IntegrityError:
799 except sqlite3.IntegrityError:
803 print("!! Session/line number for output was not unique",
800 print("!! Session/line number for output was not unique",
804 "in database. Output will not be stored.")
801 "in database. Output will not be stored.")
805 finally:
802 finally:
806 self.db_output_cache = []
803 self.db_output_cache = []
807
804
808
805
809 class HistorySavingThread(threading.Thread):
806 class HistorySavingThread(threading.Thread):
810 """This thread takes care of writing history to the database, so that
807 """This thread takes care of writing history to the database, so that
811 the UI isn't held up while that happens.
808 the UI isn't held up while that happens.
812
809
813 It waits for the HistoryManager's save_flag to be set, then writes out
810 It waits for the HistoryManager's save_flag to be set, then writes out
814 the history cache. The main thread is responsible for setting the flag when
811 the history cache. The main thread is responsible for setting the flag when
815 the cache size reaches a defined threshold."""
812 the cache size reaches a defined threshold."""
816 daemon = True
813 daemon = True
817 stop_now = False
814 stop_now = False
818 enabled = True
815 enabled = True
819 def __init__(self, history_manager):
816 def __init__(self, history_manager):
820 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
817 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
821 self.history_manager = history_manager
818 self.history_manager = history_manager
822 self.enabled = history_manager.enabled
819 self.enabled = history_manager.enabled
823 atexit.register(self.stop)
820 atexit.register(self.stop)
824
821
825 @needs_sqlite
822 @needs_sqlite
826 def run(self):
823 def run(self):
827 # We need a separate db connection per thread:
824 # We need a separate db connection per thread:
828 try:
825 try:
829 self.db = sqlite3.connect(self.history_manager.hist_file,
826 self.db = sqlite3.connect(self.history_manager.hist_file,
830 **self.history_manager.connection_options
827 **self.history_manager.connection_options
831 )
828 )
832 while True:
829 while True:
833 self.history_manager.save_flag.wait()
830 self.history_manager.save_flag.wait()
834 if self.stop_now:
831 if self.stop_now:
835 self.db.close()
832 self.db.close()
836 return
833 return
837 self.history_manager.save_flag.clear()
834 self.history_manager.save_flag.clear()
838 self.history_manager.writeout_cache(self.db)
835 self.history_manager.writeout_cache(self.db)
839 except Exception as e:
836 except Exception as e:
840 print(("The history saving thread hit an unexpected error (%s)."
837 print(("The history saving thread hit an unexpected error (%s)."
841 "History will not be written to the database.") % repr(e))
838 "History will not be written to the database.") % repr(e))
842
839
843 def stop(self):
840 def stop(self):
844 """This can be called from the main thread to safely stop this thread.
841 """This can be called from the main thread to safely stop this thread.
845
842
846 Note that it does not attempt to write out remaining history before
843 Note that it does not attempt to write out remaining history before
847 exiting. That should be done by calling the HistoryManager's
844 exiting. That should be done by calling the HistoryManager's
848 end_session method."""
845 end_session method."""
849 self.stop_now = True
846 self.stop_now = True
850 self.history_manager.save_flag.set()
847 self.history_manager.save_flag.set()
851 self.join()
848 self.join()
852
849
853
850
854 # To match, e.g. ~5/8-~2/3
851 # To match, e.g. ~5/8-~2/3
855 range_re = re.compile(r"""
852 range_re = re.compile(r"""
856 ((?P<startsess>~?\d+)/)?
853 ((?P<startsess>~?\d+)/)?
857 (?P<start>\d+)?
854 (?P<start>\d+)?
858 ((?P<sep>[\-:])
855 ((?P<sep>[\-:])
859 ((?P<endsess>~?\d+)/)?
856 ((?P<endsess>~?\d+)/)?
860 (?P<end>\d+))?
857 (?P<end>\d+))?
861 $""", re.VERBOSE)
858 $""", re.VERBOSE)
862
859
863
860
864 def extract_hist_ranges(ranges_str):
861 def extract_hist_ranges(ranges_str):
865 """Turn a string of history ranges into 3-tuples of (session, start, stop).
862 """Turn a string of history ranges into 3-tuples of (session, start, stop).
866
863
867 Examples
864 Examples
868 --------
865 --------
869 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
866 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
870 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
867 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
871 """
868 """
872 for range_str in ranges_str.split():
869 for range_str in ranges_str.split():
873 rmatch = range_re.match(range_str)
870 rmatch = range_re.match(range_str)
874 if not rmatch:
871 if not rmatch:
875 continue
872 continue
876 start = rmatch.group("start")
873 start = rmatch.group("start")
877 if start:
874 if start:
878 start = int(start)
875 start = int(start)
879 end = rmatch.group("end")
876 end = rmatch.group("end")
880 # If no end specified, get (a, a + 1)
877 # If no end specified, get (a, a + 1)
881 end = int(end) if end else start + 1
878 end = int(end) if end else start + 1
882 else: # start not specified
879 else: # start not specified
883 if not rmatch.group('startsess'): # no startsess
880 if not rmatch.group('startsess'): # no startsess
884 continue
881 continue
885 start = 1
882 start = 1
886 end = None # provide the entire session hist
883 end = None # provide the entire session hist
887
884
888 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
885 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
889 end += 1
886 end += 1
890 startsess = rmatch.group("startsess") or "0"
887 startsess = rmatch.group("startsess") or "0"
891 endsess = rmatch.group("endsess") or startsess
888 endsess = rmatch.group("endsess") or startsess
892 startsess = int(startsess.replace("~","-"))
889 startsess = int(startsess.replace("~","-"))
893 endsess = int(endsess.replace("~","-"))
890 endsess = int(endsess.replace("~","-"))
894 assert endsess >= startsess, "start session must be earlier than end session"
891 assert endsess >= startsess, "start session must be earlier than end session"
895
892
896 if endsess == startsess:
893 if endsess == startsess:
897 yield (startsess, start, end)
894 yield (startsess, start, end)
898 continue
895 continue
899 # Multiple sessions in one range:
896 # Multiple sessions in one range:
900 yield (startsess, start, None)
897 yield (startsess, start, None)
901 for sess in range(startsess+1, endsess):
898 for sess in range(startsess+1, endsess):
902 yield (sess, 1, None)
899 yield (sess, 1, None)
903 yield (endsess, 1, end)
900 yield (endsess, 1, end)
904
901
905
902
906 def _format_lineno(session, line):
903 def _format_lineno(session, line):
907 """Helper function to format line numbers properly."""
904 """Helper function to format line numbers properly."""
908 if session == 0:
905 if session == 0:
909 return str(line)
906 return str(line)
910 return "%s#%s" % (session, line)
907 return "%s#%s" % (session, line)
General Comments 0
You need to be logged in to leave comments. Login now