##// END OF EJS Templates
Update history.py to use pathlib...
Jakub Klus -
r26129:948a79c2
parent child
Show More
@@ -1,881 +1,885
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6
6
7 import atexit
7 import atexit
8 import datetime
8 import datetime
9 import os
9 from pathlib import Path
10 import re
10 import re
11 import sqlite3
11 import sqlite3
12 import threading
12 import threading
13
13
14 from traitlets.config.configurable import LoggingConfigurable
14 from traitlets.config.configurable import LoggingConfigurable
15 from decorator import decorator
15 from decorator import decorator
16 from IPython.utils.decorators import undoc
16 from IPython.utils.decorators import undoc
17 from IPython.paths import locate_profile
17 from IPython.paths import locate_profile
18 from traitlets import (
18 from traitlets import (
19 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
19 Any, Bool, Dict, Instance, Integer, List, Unicode, Union, TraitError,
20 default, observe,
20 default, observe
21 )
21 )
22
22
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24 # Classes and functions
24 # Classes and functions
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 @undoc
27 @undoc
28 class DummyDB(object):
28 class DummyDB(object):
29 """Dummy DB that will act as a black hole for history.
29 """Dummy DB that will act as a black hole for history.
30
30
31 Only used in the absence of sqlite"""
31 Only used in the absence of sqlite"""
32 def execute(*args, **kwargs):
32 def execute(*args, **kwargs):
33 return []
33 return []
34
34
35 def commit(self, *args, **kwargs):
35 def commit(self, *args, **kwargs):
36 pass
36 pass
37
37
38 def __enter__(self, *args, **kwargs):
38 def __enter__(self, *args, **kwargs):
39 pass
39 pass
40
40
41 def __exit__(self, *args, **kwargs):
41 def __exit__(self, *args, **kwargs):
42 pass
42 pass
43
43
44
44
45 @decorator
45 @decorator
46 def only_when_enabled(f, self, *a, **kw):
46 def only_when_enabled(f, self, *a, **kw):
47 """Decorator: return an empty list in the absence of sqlite."""
47 """Decorator: return an empty list in the absence of sqlite."""
48 if not self.enabled:
48 if not self.enabled:
49 return []
49 return []
50 else:
50 else:
51 return f(self, *a, **kw)
51 return f(self, *a, **kw)
52
52
53
53
54 # use 16kB as threshold for whether a corrupt history db should be saved
54 # use 16kB as threshold for whether a corrupt history db should be saved
55 # that should be at least 100 entries or so
55 # that should be at least 100 entries or so
56 _SAVE_DB_SIZE = 16384
56 _SAVE_DB_SIZE = 16384
57
57
58 @decorator
58 @decorator
59 def catch_corrupt_db(f, self, *a, **kw):
59 def catch_corrupt_db(f, self, *a, **kw):
60 """A decorator which wraps HistoryAccessor method calls to catch errors from
60 """A decorator which wraps HistoryAccessor method calls to catch errors from
61 a corrupt SQLite database, move the old database out of the way, and create
61 a corrupt SQLite database, move the old database out of the way, and create
62 a new one.
62 a new one.
63
63
64 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
64 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
65 not just a corrupt file.
65 not just a corrupt file.
66 """
66 """
67 try:
67 try:
68 return f(self, *a, **kw)
68 return f(self, *a, **kw)
69 except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
69 except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
70 self._corrupt_db_counter += 1
70 self._corrupt_db_counter += 1
71 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
71 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
72 if self.hist_file != ':memory:':
72 if self.hist_file != ':memory:':
73 if self._corrupt_db_counter > self._corrupt_db_limit:
73 if self._corrupt_db_counter > self._corrupt_db_limit:
74 self.hist_file = ':memory:'
74 self.hist_file = ':memory:'
75 self.log.error("Failed to load history too many times, history will not be saved.")
75 self.log.error("Failed to load history too many times, history will not be saved.")
76 elif os.path.isfile(self.hist_file):
76 elif self.hist_file.is_file():
77 # move the file out of the way
77 # move the file out of the way
78 base, ext = os.path.splitext(self.hist_file)
78 base = str(self.hist_file.parent / self.hist_file.stem)
79 size = os.stat(self.hist_file).st_size
79 ext = self.hist_file.suffix
80 size = self.hist_file.stat().st_size
80 if size >= _SAVE_DB_SIZE:
81 if size >= _SAVE_DB_SIZE:
81 # if there's significant content, avoid clobbering
82 # if there's significant content, avoid clobbering
82 now = datetime.datetime.now().isoformat().replace(':', '.')
83 now = datetime.datetime.now().isoformat().replace(':', '.')
83 newpath = base + '-corrupt-' + now + ext
84 newpath = base + '-corrupt-' + now + ext
84 # don't clobber previous corrupt backups
85 # don't clobber previous corrupt backups
85 for i in range(100):
86 for i in range(100):
86 if not os.path.isfile(newpath):
87 if not Path(newpath).exists():
87 break
88 break
88 else:
89 else:
89 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
90 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
90 else:
91 else:
91 # not much content, possibly empty; don't worry about clobbering
92 # not much content, possibly empty; don't worry about clobbering
92 # maybe we should just delete it?
93 # maybe we should just delete it?
93 newpath = base + '-corrupt' + ext
94 newpath = base + '-corrupt' + ext
94 os.rename(self.hist_file, newpath)
95 self.hist_file.rename(newpath)
95 self.log.error("History file was moved to %s and a new file created.", newpath)
96 self.log.error("History file was moved to %s and a new file created.", newpath)
96 self.init_db()
97 self.init_db()
97 return []
98 return []
98 else:
99 else:
99 # Failed with :memory:, something serious is wrong
100 # Failed with :memory:, something serious is wrong
100 raise
101 raise
101
102
102 class HistoryAccessorBase(LoggingConfigurable):
103 class HistoryAccessorBase(LoggingConfigurable):
103 """An abstract class for History Accessors """
104 """An abstract class for History Accessors """
104
105
105 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
106 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
106 raise NotImplementedError
107 raise NotImplementedError
107
108
108 def search(self, pattern="*", raw=True, search_raw=True,
109 def search(self, pattern="*", raw=True, search_raw=True,
109 output=False, n=None, unique=False):
110 output=False, n=None, unique=False):
110 raise NotImplementedError
111 raise NotImplementedError
111
112
112 def get_range(self, session, start=1, stop=None, raw=True,output=False):
113 def get_range(self, session, start=1, stop=None, raw=True,output=False):
113 raise NotImplementedError
114 raise NotImplementedError
114
115
115 def get_range_by_str(self, rangestr, raw=True, output=False):
116 def get_range_by_str(self, rangestr, raw=True, output=False):
116 raise NotImplementedError
117 raise NotImplementedError
117
118
118
119
119 class HistoryAccessor(HistoryAccessorBase):
120 class HistoryAccessor(HistoryAccessorBase):
120 """Access the history database without adding to it.
121 """Access the history database without adding to it.
121
122
122 This is intended for use by standalone history tools. IPython shells use
123 This is intended for use by standalone history tools. IPython shells use
123 HistoryManager, below, which is a subclass of this."""
124 HistoryManager, below, which is a subclass of this."""
124
125
125 # counter for init_db retries, so we don't keep trying over and over
126 # counter for init_db retries, so we don't keep trying over and over
126 _corrupt_db_counter = 0
127 _corrupt_db_counter = 0
127 # after two failures, fallback on :memory:
128 # after two failures, fallback on :memory:
128 _corrupt_db_limit = 2
129 _corrupt_db_limit = 2
129
130
130 # String holding the path to the history file
131 # String holding the path to the history file
131 hist_file = Unicode(
132 hist_file = Union([Instance(Path), Unicode()],
132 help="""Path to file to use for SQLite history database.
133 help="""Path to file to use for SQLite history database.
133
134
134 By default, IPython will put the history database in the IPython
135 By default, IPython will put the history database in the IPython
135 profile directory. If you would rather share one history among
136 profile directory. If you would rather share one history among
136 profiles, you can set this value in each, so that they are consistent.
137 profiles, you can set this value in each, so that they are consistent.
137
138
138 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
139 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
139 mounts. If you see IPython hanging, try setting this to something on a
140 mounts. If you see IPython hanging, try setting this to something on a
140 local disk, e.g::
141 local disk, e.g::
141
142
142 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
143 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
143
144
144 you can also use the specific value `:memory:` (including the colon
145 you can also use the specific value `:memory:` (including the colon
145 at both end but not the back ticks), to avoid creating an history file.
146 at both end but not the back ticks), to avoid creating an history file.
146
147
147 """).tag(config=True)
148 """
149 ).tag(config=True)
148
150
149 enabled = Bool(True,
151 enabled = Bool(True,
150 help="""enable the SQLite history
152 help="""enable the SQLite history
151
153
152 set enabled=False to disable the SQLite history,
154 set enabled=False to disable the SQLite history,
153 in which case there will be no stored history, no SQLite connection,
155 in which case there will be no stored history, no SQLite connection,
154 and no background saving thread. This may be necessary in some
156 and no background saving thread. This may be necessary in some
155 threaded environments where IPython is embedded.
157 threaded environments where IPython is embedded.
156 """
158 """
157 ).tag(config=True)
159 ).tag(config=True)
158
160
159 connection_options = Dict(
161 connection_options = Dict(
160 help="""Options for configuring the SQLite connection
162 help="""Options for configuring the SQLite connection
161
163
162 These options are passed as keyword args to sqlite3.connect
164 These options are passed as keyword args to sqlite3.connect
163 when establishing database connections.
165 when establishing database connections.
164 """
166 """
165 ).tag(config=True)
167 ).tag(config=True)
166
168
167 # The SQLite database
169 # The SQLite database
168 db = Any()
170 db = Any()
169 @observe('db')
171 @observe('db')
170 def _db_changed(self, change):
172 def _db_changed(self, change):
171 """validate the db, since it can be an Instance of two different types"""
173 """validate the db, since it can be an Instance of two different types"""
172 new = change['new']
174 new = change['new']
173 connection_types = (DummyDB, sqlite3.Connection)
175 connection_types = (DummyDB, sqlite3.Connection)
174 if not isinstance(new, connection_types):
176 if not isinstance(new, connection_types):
175 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
177 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
176 (self.__class__.__name__, new)
178 (self.__class__.__name__, new)
177 raise TraitError(msg)
179 raise TraitError(msg)
178
180
179 def __init__(self, profile='default', hist_file=u'', **traits):
181 def __init__(self, profile='default', hist_file="", **traits):
180 """Create a new history accessor.
182 """Create a new history accessor.
181
183
182 Parameters
184 Parameters
183 ----------
185 ----------
184 profile : str
186 profile : str
185 The name of the profile from which to open history.
187 The name of the profile from which to open history.
186 hist_file : str
188 hist_file : str
187 Path to an SQLite history database stored by IPython. If specified,
189 Path to an SQLite history database stored by IPython. If specified,
188 hist_file overrides profile.
190 hist_file overrides profile.
189 config : :class:`~traitlets.config.loader.Config`
191 config : :class:`~traitlets.config.loader.Config`
190 Config object. hist_file can also be set through this.
192 Config object. hist_file can also be set through this.
191 """
193 """
192 # We need a pointer back to the shell for various tasks.
194 # We need a pointer back to the shell for various tasks.
193 super(HistoryAccessor, self).__init__(**traits)
195 super(HistoryAccessor, self).__init__(**traits)
194 # defer setting hist_file from kwarg until after init,
196 # defer setting hist_file from kwarg until after init,
195 # otherwise the default kwarg value would clobber any value
197 # otherwise the default kwarg value would clobber any value
196 # set by config
198 # set by config
197 if hist_file:
199 if hist_file:
198 self.hist_file = hist_file
200 self.hist_file = hist_file
199
201
200 if self.hist_file == u'':
202 try:
203 self.hist_file
204 except TraitError:
201 # No one has set the hist_file, yet.
205 # No one has set the hist_file, yet.
202 self.hist_file = self._get_hist_file_name(profile)
206 self.hist_file = self._get_hist_file_name(profile)
203
207
204 self.init_db()
208 self.init_db()
205
209
206 def _get_hist_file_name(self, profile='default'):
210 def _get_hist_file_name(self, profile='default'):
207 """Find the history file for the given profile name.
211 """Find the history file for the given profile name.
208
212
209 This is overridden by the HistoryManager subclass, to use the shell's
213 This is overridden by the HistoryManager subclass, to use the shell's
210 active profile.
214 active profile.
211
215
212 Parameters
216 Parameters
213 ----------
217 ----------
214 profile : str
218 profile : str
215 The name of a profile which has a history file.
219 The name of a profile which has a history file.
216 """
220 """
217 return os.path.join(locate_profile(profile), 'history.sqlite')
221 return Path(locate_profile(profile)) / 'history.sqlite'
218
222
219 @catch_corrupt_db
223 @catch_corrupt_db
220 def init_db(self):
224 def init_db(self):
221 """Connect to the database, and create tables if necessary."""
225 """Connect to the database, and create tables if necessary."""
222 if not self.enabled:
226 if not self.enabled:
223 self.db = DummyDB()
227 self.db = DummyDB()
224 return
228 return
225
229
226 # use detect_types so that timestamps return datetime objects
230 # use detect_types so that timestamps return datetime objects
227 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
231 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
228 kwargs.update(self.connection_options)
232 kwargs.update(self.connection_options)
229 self.db = sqlite3.connect(self.hist_file, **kwargs)
233 self.db = sqlite3.connect(self.hist_file, **kwargs)
230 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
234 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
231 primary key autoincrement, start timestamp,
235 primary key autoincrement, start timestamp,
232 end timestamp, num_cmds integer, remark text)""")
236 end timestamp, num_cmds integer, remark text)""")
233 self.db.execute("""CREATE TABLE IF NOT EXISTS history
237 self.db.execute("""CREATE TABLE IF NOT EXISTS history
234 (session integer, line integer, source text, source_raw text,
238 (session integer, line integer, source text, source_raw text,
235 PRIMARY KEY (session, line))""")
239 PRIMARY KEY (session, line))""")
236 # Output history is optional, but ensure the table's there so it can be
240 # Output history is optional, but ensure the table's there so it can be
237 # enabled later.
241 # enabled later.
238 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
242 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
239 (session integer, line integer, output text,
243 (session integer, line integer, output text,
240 PRIMARY KEY (session, line))""")
244 PRIMARY KEY (session, line))""")
241 self.db.commit()
245 self.db.commit()
242 # success! reset corrupt db count
246 # success! reset corrupt db count
243 self._corrupt_db_counter = 0
247 self._corrupt_db_counter = 0
244
248
245 def writeout_cache(self):
249 def writeout_cache(self):
246 """Overridden by HistoryManager to dump the cache before certain
250 """Overridden by HistoryManager to dump the cache before certain
247 database lookups."""
251 database lookups."""
248 pass
252 pass
249
253
250 ## -------------------------------
254 ## -------------------------------
251 ## Methods for retrieving history:
255 ## Methods for retrieving history:
252 ## -------------------------------
256 ## -------------------------------
253 def _run_sql(self, sql, params, raw=True, output=False):
257 def _run_sql(self, sql, params, raw=True, output=False):
254 """Prepares and runs an SQL query for the history database.
258 """Prepares and runs an SQL query for the history database.
255
259
256 Parameters
260 Parameters
257 ----------
261 ----------
258 sql : str
262 sql : str
259 Any filtering expressions to go after SELECT ... FROM ...
263 Any filtering expressions to go after SELECT ... FROM ...
260 params : tuple
264 params : tuple
261 Parameters passed to the SQL query (to replace "?")
265 Parameters passed to the SQL query (to replace "?")
262 raw, output : bool
266 raw, output : bool
263 See :meth:`get_range`
267 See :meth:`get_range`
264
268
265 Returns
269 Returns
266 -------
270 -------
267 Tuples as :meth:`get_range`
271 Tuples as :meth:`get_range`
268 """
272 """
269 toget = 'source_raw' if raw else 'source'
273 toget = 'source_raw' if raw else 'source'
270 sqlfrom = "history"
274 sqlfrom = "history"
271 if output:
275 if output:
272 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
276 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
273 toget = "history.%s, output_history.output" % toget
277 toget = "history.%s, output_history.output" % toget
274 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
278 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
275 (toget, sqlfrom) + sql, params)
279 (toget, sqlfrom) + sql, params)
276 if output: # Regroup into 3-tuples, and parse JSON
280 if output: # Regroup into 3-tuples, and parse JSON
277 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
281 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
278 return cur
282 return cur
279
283
280 @only_when_enabled
284 @only_when_enabled
281 @catch_corrupt_db
285 @catch_corrupt_db
282 def get_session_info(self, session):
286 def get_session_info(self, session):
283 """Get info about a session.
287 """Get info about a session.
284
288
285 Parameters
289 Parameters
286 ----------
290 ----------
287
291
288 session : int
292 session : int
289 Session number to retrieve.
293 Session number to retrieve.
290
294
291 Returns
295 Returns
292 -------
296 -------
293
297
294 session_id : int
298 session_id : int
295 Session ID number
299 Session ID number
296 start : datetime
300 start : datetime
297 Timestamp for the start of the session.
301 Timestamp for the start of the session.
298 end : datetime
302 end : datetime
299 Timestamp for the end of the session, or None if IPython crashed.
303 Timestamp for the end of the session, or None if IPython crashed.
300 num_cmds : int
304 num_cmds : int
301 Number of commands run, or None if IPython crashed.
305 Number of commands run, or None if IPython crashed.
302 remark : unicode
306 remark : unicode
303 A manually set description.
307 A manually set description.
304 """
308 """
305 query = "SELECT * from sessions where session == ?"
309 query = "SELECT * from sessions where session == ?"
306 return self.db.execute(query, (session,)).fetchone()
310 return self.db.execute(query, (session,)).fetchone()
307
311
308 @catch_corrupt_db
312 @catch_corrupt_db
309 def get_last_session_id(self):
313 def get_last_session_id(self):
310 """Get the last session ID currently in the database.
314 """Get the last session ID currently in the database.
311
315
312 Within IPython, this should be the same as the value stored in
316 Within IPython, this should be the same as the value stored in
313 :attr:`HistoryManager.session_number`.
317 :attr:`HistoryManager.session_number`.
314 """
318 """
315 for record in self.get_tail(n=1, include_latest=True):
319 for record in self.get_tail(n=1, include_latest=True):
316 return record[0]
320 return record[0]
317
321
318 @catch_corrupt_db
322 @catch_corrupt_db
319 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
323 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
320 """Get the last n lines from the history database.
324 """Get the last n lines from the history database.
321
325
322 Parameters
326 Parameters
323 ----------
327 ----------
324 n : int
328 n : int
325 The number of lines to get
329 The number of lines to get
326 raw, output : bool
330 raw, output : bool
327 See :meth:`get_range`
331 See :meth:`get_range`
328 include_latest : bool
332 include_latest : bool
329 If False (default), n+1 lines are fetched, and the latest one
333 If False (default), n+1 lines are fetched, and the latest one
330 is discarded. This is intended to be used where the function
334 is discarded. This is intended to be used where the function
331 is called by a user command, which it should not return.
335 is called by a user command, which it should not return.
332
336
333 Returns
337 Returns
334 -------
338 -------
335 Tuples as :meth:`get_range`
339 Tuples as :meth:`get_range`
336 """
340 """
337 self.writeout_cache()
341 self.writeout_cache()
338 if not include_latest:
342 if not include_latest:
339 n += 1
343 n += 1
340 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
344 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
341 (n,), raw=raw, output=output)
345 (n,), raw=raw, output=output)
342 if not include_latest:
346 if not include_latest:
343 return reversed(list(cur)[1:])
347 return reversed(list(cur)[1:])
344 return reversed(list(cur))
348 return reversed(list(cur))
345
349
346 @catch_corrupt_db
350 @catch_corrupt_db
347 def search(self, pattern="*", raw=True, search_raw=True,
351 def search(self, pattern="*", raw=True, search_raw=True,
348 output=False, n=None, unique=False):
352 output=False, n=None, unique=False):
349 """Search the database using unix glob-style matching (wildcards
353 """Search the database using unix glob-style matching (wildcards
350 * and ?).
354 * and ?).
351
355
352 Parameters
356 Parameters
353 ----------
357 ----------
354 pattern : str
358 pattern : str
355 The wildcarded pattern to match when searching
359 The wildcarded pattern to match when searching
356 search_raw : bool
360 search_raw : bool
357 If True, search the raw input, otherwise, the parsed input
361 If True, search the raw input, otherwise, the parsed input
358 raw, output : bool
362 raw, output : bool
359 See :meth:`get_range`
363 See :meth:`get_range`
360 n : None or int
364 n : None or int
361 If an integer is given, it defines the limit of
365 If an integer is given, it defines the limit of
362 returned entries.
366 returned entries.
363 unique : bool
367 unique : bool
364 When it is true, return only unique entries.
368 When it is true, return only unique entries.
365
369
366 Returns
370 Returns
367 -------
371 -------
368 Tuples as :meth:`get_range`
372 Tuples as :meth:`get_range`
369 """
373 """
370 tosearch = "source_raw" if search_raw else "source"
374 tosearch = "source_raw" if search_raw else "source"
371 if output:
375 if output:
372 tosearch = "history." + tosearch
376 tosearch = "history." + tosearch
373 self.writeout_cache()
377 self.writeout_cache()
374 sqlform = "WHERE %s GLOB ?" % tosearch
378 sqlform = "WHERE %s GLOB ?" % tosearch
375 params = (pattern,)
379 params = (pattern,)
376 if unique:
380 if unique:
377 sqlform += ' GROUP BY {0}'.format(tosearch)
381 sqlform += ' GROUP BY {0}'.format(tosearch)
378 if n is not None:
382 if n is not None:
379 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
383 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
380 params += (n,)
384 params += (n,)
381 elif unique:
385 elif unique:
382 sqlform += " ORDER BY session, line"
386 sqlform += " ORDER BY session, line"
383 cur = self._run_sql(sqlform, params, raw=raw, output=output)
387 cur = self._run_sql(sqlform, params, raw=raw, output=output)
384 if n is not None:
388 if n is not None:
385 return reversed(list(cur))
389 return reversed(list(cur))
386 return cur
390 return cur
387
391
388 @catch_corrupt_db
392 @catch_corrupt_db
389 def get_range(self, session, start=1, stop=None, raw=True,output=False):
393 def get_range(self, session, start=1, stop=None, raw=True,output=False):
390 """Retrieve input by session.
394 """Retrieve input by session.
391
395
392 Parameters
396 Parameters
393 ----------
397 ----------
394 session : int
398 session : int
395 Session number to retrieve.
399 Session number to retrieve.
396 start : int
400 start : int
397 First line to retrieve.
401 First line to retrieve.
398 stop : int
402 stop : int
399 End of line range (excluded from output itself). If None, retrieve
403 End of line range (excluded from output itself). If None, retrieve
400 to the end of the session.
404 to the end of the session.
401 raw : bool
405 raw : bool
402 If True, return untranslated input
406 If True, return untranslated input
403 output : bool
407 output : bool
404 If True, attempt to include output. This will be 'real' Python
408 If True, attempt to include output. This will be 'real' Python
405 objects for the current session, or text reprs from previous
409 objects for the current session, or text reprs from previous
406 sessions if db_log_output was enabled at the time. Where no output
410 sessions if db_log_output was enabled at the time. Where no output
407 is found, None is used.
411 is found, None is used.
408
412
409 Returns
413 Returns
410 -------
414 -------
411 entries
415 entries
412 An iterator over the desired lines. Each line is a 3-tuple, either
416 An iterator over the desired lines. Each line is a 3-tuple, either
413 (session, line, input) if output is False, or
417 (session, line, input) if output is False, or
414 (session, line, (input, output)) if output is True.
418 (session, line, (input, output)) if output is True.
415 """
419 """
416 if stop:
420 if stop:
417 lineclause = "line >= ? AND line < ?"
421 lineclause = "line >= ? AND line < ?"
418 params = (session, start, stop)
422 params = (session, start, stop)
419 else:
423 else:
420 lineclause = "line>=?"
424 lineclause = "line>=?"
421 params = (session, start)
425 params = (session, start)
422
426
423 return self._run_sql("WHERE session==? AND %s" % lineclause,
427 return self._run_sql("WHERE session==? AND %s" % lineclause,
424 params, raw=raw, output=output)
428 params, raw=raw, output=output)
425
429
426 def get_range_by_str(self, rangestr, raw=True, output=False):
430 def get_range_by_str(self, rangestr, raw=True, output=False):
427 """Get lines of history from a string of ranges, as used by magic
431 """Get lines of history from a string of ranges, as used by magic
428 commands %hist, %save, %macro, etc.
432 commands %hist, %save, %macro, etc.
429
433
430 Parameters
434 Parameters
431 ----------
435 ----------
432 rangestr : str
436 rangestr : str
433 A string specifying ranges, e.g. "5 ~2/1-4". See
437 A string specifying ranges, e.g. "5 ~2/1-4". See
434 :func:`magic_history` for full details.
438 :func:`magic_history` for full details.
435 raw, output : bool
439 raw, output : bool
436 As :meth:`get_range`
440 As :meth:`get_range`
437
441
438 Returns
442 Returns
439 -------
443 -------
440 Tuples as :meth:`get_range`
444 Tuples as :meth:`get_range`
441 """
445 """
442 for sess, s, e in extract_hist_ranges(rangestr):
446 for sess, s, e in extract_hist_ranges(rangestr):
443 for line in self.get_range(sess, s, e, raw=raw, output=output):
447 for line in self.get_range(sess, s, e, raw=raw, output=output):
444 yield line
448 yield line
445
449
446
450
447 class HistoryManager(HistoryAccessor):
451 class HistoryManager(HistoryAccessor):
448 """A class to organize all history-related functionality in one place.
452 """A class to organize all history-related functionality in one place.
449 """
453 """
450 # Public interface
454 # Public interface
451
455
452 # An instance of the IPython shell we are attached to
456 # An instance of the IPython shell we are attached to
453 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
457 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
454 allow_none=True)
458 allow_none=True)
455 # Lists to hold processed and raw history. These start with a blank entry
459 # Lists to hold processed and raw history. These start with a blank entry
456 # so that we can index them starting from 1
460 # so that we can index them starting from 1
457 input_hist_parsed = List([""])
461 input_hist_parsed = List([""])
458 input_hist_raw = List([""])
462 input_hist_raw = List([""])
459 # A list of directories visited during session
463 # A list of directories visited during session
460 dir_hist = List()
464 dir_hist = List()
461 @default('dir_hist')
465 @default('dir_hist')
462 def _dir_hist_default(self):
466 def _dir_hist_default(self):
463 try:
467 try:
464 return [os.getcwd()]
468 return [Path.cwd()]
465 except OSError:
469 except OSError:
466 return []
470 return []
467
471
468 # A dict of output history, keyed with ints from the shell's
472 # A dict of output history, keyed with ints from the shell's
469 # execution count.
473 # execution count.
470 output_hist = Dict()
474 output_hist = Dict()
471 # The text/plain repr of outputs.
475 # The text/plain repr of outputs.
472 output_hist_reprs = Dict()
476 output_hist_reprs = Dict()
473
477
474 # The number of the current session in the history database
478 # The number of the current session in the history database
475 session_number = Integer()
479 session_number = Integer()
476
480
477 db_log_output = Bool(False,
481 db_log_output = Bool(False,
478 help="Should the history database include output? (default: no)"
482 help="Should the history database include output? (default: no)"
479 ).tag(config=True)
483 ).tag(config=True)
480 db_cache_size = Integer(0,
484 db_cache_size = Integer(0,
481 help="Write to database every x commands (higher values save disk access & power).\n"
485 help="Write to database every x commands (higher values save disk access & power).\n"
482 "Values of 1 or less effectively disable caching."
486 "Values of 1 or less effectively disable caching."
483 ).tag(config=True)
487 ).tag(config=True)
484 # The input and output caches
488 # The input and output caches
485 db_input_cache = List()
489 db_input_cache = List()
486 db_output_cache = List()
490 db_output_cache = List()
487
491
488 # History saving in separate thread
492 # History saving in separate thread
489 save_thread = Instance('IPython.core.history.HistorySavingThread',
493 save_thread = Instance('IPython.core.history.HistorySavingThread',
490 allow_none=True)
494 allow_none=True)
491 save_flag = Instance(threading.Event, allow_none=True)
495 save_flag = Instance(threading.Event, allow_none=True)
492
496
493 # Private interface
497 # Private interface
494 # Variables used to store the three last inputs from the user. On each new
498 # Variables used to store the three last inputs from the user. On each new
495 # history update, we populate the user's namespace with these, shifted as
499 # history update, we populate the user's namespace with these, shifted as
496 # necessary.
500 # necessary.
497 _i00 = Unicode(u'')
501 _i00 = Unicode(u'')
498 _i = Unicode(u'')
502 _i = Unicode(u'')
499 _ii = Unicode(u'')
503 _ii = Unicode(u'')
500 _iii = Unicode(u'')
504 _iii = Unicode(u'')
501
505
502 # A regex matching all forms of the exit command, so that we don't store
506 # A regex matching all forms of the exit command, so that we don't store
503 # them in the history (it's annoying to rewind the first entry and land on
507 # them in the history (it's annoying to rewind the first entry and land on
504 # an exit call).
508 # an exit call).
505 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
509 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
506
510
507 def __init__(self, shell=None, config=None, **traits):
511 def __init__(self, shell=None, config=None, **traits):
508 """Create a new history manager associated with a shell instance.
512 """Create a new history manager associated with a shell instance.
509 """
513 """
510 # We need a pointer back to the shell for various tasks.
514 # We need a pointer back to the shell for various tasks.
511 super(HistoryManager, self).__init__(shell=shell, config=config,
515 super(HistoryManager, self).__init__(shell=shell, config=config,
512 **traits)
516 **traits)
513 self.save_flag = threading.Event()
517 self.save_flag = threading.Event()
514 self.db_input_cache_lock = threading.Lock()
518 self.db_input_cache_lock = threading.Lock()
515 self.db_output_cache_lock = threading.Lock()
519 self.db_output_cache_lock = threading.Lock()
516
520
517 try:
521 try:
518 self.new_session()
522 self.new_session()
519 except sqlite3.OperationalError:
523 except sqlite3.OperationalError:
520 self.log.error("Failed to create history session in %s. History will not be saved.",
524 self.log.error("Failed to create history session in %s. History will not be saved.",
521 self.hist_file, exc_info=True)
525 self.hist_file, exc_info=True)
522 self.hist_file = ':memory:'
526 self.hist_file = ':memory:'
523
527
524 if self.enabled and self.hist_file != ':memory:':
528 if self.enabled and self.hist_file != ':memory:':
525 self.save_thread = HistorySavingThread(self)
529 self.save_thread = HistorySavingThread(self)
526 self.save_thread.start()
530 self.save_thread.start()
527
531
528 def _get_hist_file_name(self, profile=None):
532 def _get_hist_file_name(self, profile=None):
529 """Get default history file name based on the Shell's profile.
533 """Get default history file name based on the Shell's profile.
530
534
531 The profile parameter is ignored, but must exist for compatibility with
535 The profile parameter is ignored, but must exist for compatibility with
532 the parent class."""
536 the parent class."""
533 profile_dir = self.shell.profile_dir.location
537 profile_dir = self.shell.profile_dir.location
534 return os.path.join(profile_dir, 'history.sqlite')
538 return Path(profile_dir)/'history.sqlite'
535
539
536 @only_when_enabled
540 @only_when_enabled
537 def new_session(self, conn=None):
541 def new_session(self, conn=None):
538 """Get a new session number."""
542 """Get a new session number."""
539 if conn is None:
543 if conn is None:
540 conn = self.db
544 conn = self.db
541
545
542 with conn:
546 with conn:
543 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
547 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
544 NULL, "") """, (datetime.datetime.now(),))
548 NULL, "") """, (datetime.datetime.now(),))
545 self.session_number = cur.lastrowid
549 self.session_number = cur.lastrowid
546
550
547 def end_session(self):
551 def end_session(self):
548 """Close the database session, filling in the end time and line count."""
552 """Close the database session, filling in the end time and line count."""
549 self.writeout_cache()
553 self.writeout_cache()
550 with self.db:
554 with self.db:
551 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
555 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
552 session==?""", (datetime.datetime.now(),
556 session==?""", (datetime.datetime.now(),
553 len(self.input_hist_parsed)-1, self.session_number))
557 len(self.input_hist_parsed)-1, self.session_number))
554 self.session_number = 0
558 self.session_number = 0
555
559
556 def name_session(self, name):
560 def name_session(self, name):
557 """Give the current session a name in the history database."""
561 """Give the current session a name in the history database."""
558 with self.db:
562 with self.db:
559 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
563 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
560 (name, self.session_number))
564 (name, self.session_number))
561
565
562 def reset(self, new_session=True):
566 def reset(self, new_session=True):
563 """Clear the session history, releasing all object references, and
567 """Clear the session history, releasing all object references, and
564 optionally open a new session."""
568 optionally open a new session."""
565 self.output_hist.clear()
569 self.output_hist.clear()
566 # The directory history can't be completely empty
570 # The directory history can't be completely empty
567 self.dir_hist[:] = [os.getcwd()]
571 self.dir_hist[:] = [Path.cwd()]
568
572
569 if new_session:
573 if new_session:
570 if self.session_number:
574 if self.session_number:
571 self.end_session()
575 self.end_session()
572 self.input_hist_parsed[:] = [""]
576 self.input_hist_parsed[:] = [""]
573 self.input_hist_raw[:] = [""]
577 self.input_hist_raw[:] = [""]
574 self.new_session()
578 self.new_session()
575
579
576 # ------------------------------
580 # ------------------------------
577 # Methods for retrieving history
581 # Methods for retrieving history
578 # ------------------------------
582 # ------------------------------
579 def get_session_info(self, session=0):
583 def get_session_info(self, session=0):
580 """Get info about a session.
584 """Get info about a session.
581
585
582 Parameters
586 Parameters
583 ----------
587 ----------
584
588
585 session : int
589 session : int
586 Session number to retrieve. The current session is 0, and negative
590 Session number to retrieve. The current session is 0, and negative
587 numbers count back from current session, so -1 is the previous session.
591 numbers count back from current session, so -1 is the previous session.
588
592
589 Returns
593 Returns
590 -------
594 -------
591
595
592 session_id : int
596 session_id : int
593 Session ID number
597 Session ID number
594 start : datetime
598 start : datetime
595 Timestamp for the start of the session.
599 Timestamp for the start of the session.
596 end : datetime
600 end : datetime
597 Timestamp for the end of the session, or None if IPython crashed.
601 Timestamp for the end of the session, or None if IPython crashed.
598 num_cmds : int
602 num_cmds : int
599 Number of commands run, or None if IPython crashed.
603 Number of commands run, or None if IPython crashed.
600 remark : unicode
604 remark : unicode
601 A manually set description.
605 A manually set description.
602 """
606 """
603 if session <= 0:
607 if session <= 0:
604 session += self.session_number
608 session += self.session_number
605
609
606 return super(HistoryManager, self).get_session_info(session=session)
610 return super(HistoryManager, self).get_session_info(session=session)
607
611
608 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
612 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
609 """Get input and output history from the current session. Called by
613 """Get input and output history from the current session. Called by
610 get_range, and takes similar parameters."""
614 get_range, and takes similar parameters."""
611 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
615 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
612
616
613 n = len(input_hist)
617 n = len(input_hist)
614 if start < 0:
618 if start < 0: