##// END OF EJS Templates
Add error handling so SQLite history can recover from corrupt databases and session/line number collisions.
Thomas Kluyver -
Show More
@@ -1,709 +1,740 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2 #-----------------------------------------------------------------------------
2 #-----------------------------------------------------------------------------
3 # Copyright (C) 2010 The IPython Development Team.
3 # Copyright (C) 2010 The IPython Development Team.
4 #
4 #
5 # Distributed under the terms of the BSD License.
5 # Distributed under the terms of the BSD License.
6 #
6 #
7 # The full license is in the file COPYING.txt, distributed with this software.
7 # The full license is in the file COPYING.txt, distributed with this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 from __future__ import print_function
13 from __future__ import print_function
14
14
15 # Stdlib imports
15 # Stdlib imports
16 import datetime
16 import datetime
17 import json
17 import json
18 import os
18 import os
19 import re
19 import re
20 import sqlite3
20 import sqlite3
21
21
22 from collections import defaultdict
22 from collections import defaultdict
23
23
24 # Our own packages
24 # Our own packages
25 from IPython.config.configurable import Configurable
25 from IPython.config.configurable import Configurable
26 import IPython.utils.io
26 import IPython.utils.io
27
27
28 from IPython.testing import decorators as testdec
28 from IPython.testing import decorators as testdec
29 from IPython.utils.io import ask_yes_no
29 from IPython.utils.io import ask_yes_no
30 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
30 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
31 from IPython.utils.warn import warn
31 from IPython.utils.warn import warn
32
32
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34 # Classes and functions
34 # Classes and functions
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36
36
37 class HistoryManager(Configurable):
37 class HistoryManager(Configurable):
38 """A class to organize all history-related functionality in one place.
38 """A class to organize all history-related functionality in one place.
39 """
39 """
40 # Public interface
40 # Public interface
41
41
42 # An instance of the IPython shell we are attached to
42 # An instance of the IPython shell we are attached to
43 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
43 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
44 # Lists to hold processed and raw history. These start with a blank entry
44 # Lists to hold processed and raw history. These start with a blank entry
45 # so that we can index them starting from 1
45 # so that we can index them starting from 1
46 input_hist_parsed = List([""])
46 input_hist_parsed = List([""])
47 input_hist_raw = List([""])
47 input_hist_raw = List([""])
48 # A list of directories visited during session
48 # A list of directories visited during session
49 dir_hist = List()
49 dir_hist = List()
50 # A dict of output history, keyed with ints from the shell's
50 # A dict of output history, keyed with ints from the shell's
51 # execution count. If there are several outputs from one command,
51 # execution count. If there are several outputs from one command,
52 # only the last one is stored.
52 # only the last one is stored.
53 output_hist = Dict()
53 output_hist = Dict()
54 # Contains all outputs, in lists of reprs.
54 # Contains all outputs, in lists of reprs.
55 output_hist_reprs = Instance(defaultdict)
55 output_hist_reprs = Instance(defaultdict)
56
56
57 # String holding the path to the history file
57 # String holding the path to the history file
58 hist_file = Unicode()
58 hist_file = Unicode()
59 # The SQLite database
59 # The SQLite database
60 db = Instance(sqlite3.Connection)
60 db = Instance(sqlite3.Connection)
61 # The number of the current session in the history database
61 # The number of the current session in the history database
62 session_number = Int()
62 session_number = Int()
63 # Should we log output to the database? (default no)
63 # Should we log output to the database? (default no)
64 db_log_output = Bool(False, config=True)
64 db_log_output = Bool(False, config=True)
65 # Write to database every x commands (higher values save disk access & power)
65 # Write to database every x commands (higher values save disk access & power)
66 # Values of 1 or less effectively disable caching.
66 # Values of 1 or less effectively disable caching.
67 db_cache_size = Int(0, config=True)
67 db_cache_size = Int(0, config=True)
68 # The input and output caches
68 # The input and output caches
69 db_input_cache = List()
69 db_input_cache = List()
70 db_output_cache = List()
70 db_output_cache = List()
71
71
72 # Private interface
72 # Private interface
73 # Variables used to store the three last inputs from the user. On each new
73 # Variables used to store the three last inputs from the user. On each new
74 # history update, we populate the user's namespace with these, shifted as
74 # history update, we populate the user's namespace with these, shifted as
75 # necessary.
75 # necessary.
76 _i00, _i, _ii, _iii = '','','',''
76 _i00, _i, _ii, _iii = '','','',''
77
77
78 # A set with all forms of the exit command, so that we don't store them in
78 # A set with all forms of the exit command, so that we don't store them in
79 # the history (it's annoying to rewind the first entry and land on an exit
79 # the history (it's annoying to rewind the first entry and land on an exit
80 # call).
80 # call).
81 _exit_commands = None
81 _exit_commands = None
82
82
83 def __init__(self, shell, config=None):
83 def __init__(self, shell, config=None):
84 """Create a new history manager associated with a shell instance.
84 """Create a new history manager associated with a shell instance.
85 """
85 """
86 # We need a pointer back to the shell for various tasks.
86 # We need a pointer back to the shell for various tasks.
87 super(HistoryManager, self).__init__(shell=shell, config=config)
87 super(HistoryManager, self).__init__(shell=shell, config=config)
88
88
89 # list of visited directories
89 # list of visited directories
90 try:
90 try:
91 self.dir_hist = [os.getcwd()]
91 self.dir_hist = [os.getcwd()]
92 except OSError:
92 except OSError:
93 self.dir_hist = []
93 self.dir_hist = []
94
94
95 # Now the history file
95 # Now the history file
96 if shell.profile:
96 if shell.profile:
97 histfname = 'history-%s' % shell.profile
97 histfname = 'history-%s' % shell.profile
98 else:
98 else:
99 histfname = 'history'
99 histfname = 'history'
100 self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
100 self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
101 try:
102 self.init_db()
103 except sqlite3.DatabaseError:
104 newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
105 os.rename(self.hist_file, newpath)
106 print("ERROR! History file wasn't a valid SQLite database.",
107 "It was moved to %s" % newpath, "and a new file created.")
101 self.init_db()
108 self.init_db()
109
102 self.new_session()
110 self.new_session()
103
111
104 self._i00, self._i, self._ii, self._iii = '','','',''
112 self._i00, self._i, self._ii, self._iii = '','','',''
105 self.output_hist_reprs = defaultdict(list)
113 self.output_hist_reprs = defaultdict(list)
106
114
107 self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
115 self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
108 '%quit', '%Exit', '%exit'])
116 '%quit', '%Exit', '%exit'])
109
117
110 def init_db(self):
118 def init_db(self):
111 """Connect to the database, and create tables if necessary."""
119 """Connect to the database, and create tables if necessary."""
112 self.db = sqlite3.connect(self.hist_file)
120 self.db = sqlite3.connect(self.hist_file)
113 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
121 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
114 primary key autoincrement, start timestamp,
122 primary key autoincrement, start timestamp,
115 end timestamp, num_cmds integer, remark text)""")
123 end timestamp, num_cmds integer, remark text)""")
116 self.db.execute("""CREATE TABLE IF NOT EXISTS history
124 self.db.execute("""CREATE TABLE IF NOT EXISTS history
117 (session integer, line integer, source text, source_raw text,
125 (session integer, line integer, source text, source_raw text,
118 PRIMARY KEY (session, line))""")
126 PRIMARY KEY (session, line))""")
119 # Output history is optional, but ensure the table's there so it can be
127 # Output history is optional, but ensure the table's there so it can be
120 # enabled later.
128 # enabled later.
121 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
129 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
122 (session integer, line integer, output text,
130 (session integer, line integer, output text,
123 PRIMARY KEY (session, line))""")
131 PRIMARY KEY (session, line))""")
124 self.db.commit()
132 self.db.commit()
125
133
126 def new_session(self):
134 def new_session(self):
127 """Get a new session number."""
135 """Get a new session number."""
128 with self.db:
136 with self.db:
129 cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
137 cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
130 NULL, "") """, (datetime.datetime.now(),))
138 NULL, "") """, (datetime.datetime.now(),))
131 self.session_number = cur.lastrowid
139 self.session_number = cur.lastrowid
132
140
133 def end_session(self):
141 def end_session(self):
134 """Close the database session, filling in the end time and line count."""
142 """Close the database session, filling in the end time and line count."""
135 self.writeout_cache()
143 self.writeout_cache()
136 with self.db:
144 with self.db:
137 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
145 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
138 session==?""", (datetime.datetime.now(),
146 session==?""", (datetime.datetime.now(),
139 len(self.input_hist_parsed)-1, self.session_number))
147 len(self.input_hist_parsed)-1, self.session_number))
140 self.session_number = 0
148 self.session_number = 0
141
149
142 def name_session(self, name):
150 def name_session(self, name):
143 """Give the current session a name in the history database."""
151 """Give the current session a name in the history database."""
144 with self.db:
152 with self.db:
145 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
153 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
146 (name, self.session_number))
154 (name, self.session_number))
147
155
148 def reset(self, new_session=True):
156 def reset(self, new_session=True):
149 """Clear the session history, releasing all object references, and
157 """Clear the session history, releasing all object references, and
150 optionally open a new session."""
158 optionally open a new session."""
151 if self.session_number:
159 if self.session_number:
152 self.end_session()
160 self.end_session()
153 self.input_hist_parsed[:] = [""]
161 self.input_hist_parsed[:] = [""]
154 self.input_hist_raw[:] = [""]
162 self.input_hist_raw[:] = [""]
155 self.output_hist.clear()
163 self.output_hist.clear()
156 # The directory history can't be completely empty
164 # The directory history can't be completely empty
157 self.dir_hist[:] = [os.getcwd()]
165 self.dir_hist[:] = [os.getcwd()]
158
166
159 if new_session:
167 if new_session:
160 self.new_session()
168 self.new_session()
161
169
162 ## -------------------------------
170 ## -------------------------------
163 ## Methods for retrieving history:
171 ## Methods for retrieving history:
164 ## -------------------------------
172 ## -------------------------------
165 def _run_sql(self, sql, params, raw=True, output=False):
173 def _run_sql(self, sql, params, raw=True, output=False):
166 """Prepares and runs an SQL query for the history database.
174 """Prepares and runs an SQL query for the history database.
167
175
168 Parameters
176 Parameters
169 ----------
177 ----------
170 sql : str
178 sql : str
171 Any filtering expressions to go after SELECT ... FROM ...
179 Any filtering expressions to go after SELECT ... FROM ...
172 params : tuple
180 params : tuple
173 Parameters passed to the SQL query (to replace "?")
181 Parameters passed to the SQL query (to replace "?")
174 raw, output : bool
182 raw, output : bool
175 See :meth:`get_range`
183 See :meth:`get_range`
176
184
177 Returns
185 Returns
178 -------
186 -------
179 Tuples as :meth:`get_range`
187 Tuples as :meth:`get_range`
180 """
188 """
181 toget = 'source_raw' if raw else 'source'
189 toget = 'source_raw' if raw else 'source'
182 sqlfrom = "history"
190 sqlfrom = "history"
183 if output:
191 if output:
184 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
192 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
185 toget = "history.%s, output_history.output" % toget
193 toget = "history.%s, output_history.output" % toget
186 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
194 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
187 (toget, sqlfrom) + sql, params)
195 (toget, sqlfrom) + sql, params)
188 if output: # Regroup into 3-tuples, and parse JSON
196 if output: # Regroup into 3-tuples, and parse JSON
189 loads = lambda out: json.loads(out) if out else None
197 loads = lambda out: json.loads(out) if out else None
190 return ((ses, lin, (inp, loads(out))) \
198 return ((ses, lin, (inp, loads(out))) \
191 for ses, lin, inp, out in cur)
199 for ses, lin, inp, out in cur)
192 return cur
200 return cur
193
201
194
202
195 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
203 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
196 """Get the last n lines from the history database.
204 """Get the last n lines from the history database.
197
205
198 Parameters
206 Parameters
199 ----------
207 ----------
200 n : int
208 n : int
201 The number of lines to get
209 The number of lines to get
202 raw, output : bool
210 raw, output : bool
203 See :meth:`get_range`
211 See :meth:`get_range`
204 include_latest : bool
212 include_latest : bool
205 If False (default), n+1 lines are fetched, and the latest one
213 If False (default), n+1 lines are fetched, and the latest one
206 is discarded. This is intended to be used where the function
214 is discarded. This is intended to be used where the function
207 is called by a user command, which it should not return.
215 is called by a user command, which it should not return.
208
216
209 Returns
217 Returns
210 -------
218 -------
211 Tuples as :meth:`get_range`
219 Tuples as :meth:`get_range`
212 """
220 """
213 self.writeout_cache()
221 self.writeout_cache()
214 if not include_latest:
222 if not include_latest:
215 n += 1
223 n += 1
216 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
224 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
217 (n,), raw=raw, output=output)
225 (n,), raw=raw, output=output)
218 if not include_latest:
226 if not include_latest:
219 return reversed(list(cur)[1:])
227 return reversed(list(cur)[1:])
220 return reversed(list(cur))
228 return reversed(list(cur))
221
229
222 def search(self, pattern="*", raw=True, search_raw=True,
230 def search(self, pattern="*", raw=True, search_raw=True,
223 output=False):
231 output=False):
224 """Search the database using unix glob-style matching (wildcards
232 """Search the database using unix glob-style matching (wildcards
225 * and ?).
233 * and ?).
226
234
227 Parameters
235 Parameters
228 ----------
236 ----------
229 pattern : str
237 pattern : str
230 The wildcarded pattern to match when searching
238 The wildcarded pattern to match when searching
231 search_raw : bool
239 search_raw : bool
232 If True, search the raw input, otherwise, the parsed input
240 If True, search the raw input, otherwise, the parsed input
233 raw, output : bool
241 raw, output : bool
234 See :meth:`get_range`
242 See :meth:`get_range`
235
243
236 Returns
244 Returns
237 -------
245 -------
238 Tuples as :meth:`get_range`
246 Tuples as :meth:`get_range`
239 """
247 """
240 tosearch = "source_raw" if search_raw else "source"
248 tosearch = "source_raw" if search_raw else "source"
241 if output:
249 if output:
242 tosearch = "history." + tosearch
250 tosearch = "history." + tosearch
243 self.writeout_cache()
251 self.writeout_cache()
244 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
252 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
245 raw=raw, output=output)
253 raw=raw, output=output)
246
254
247 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
255 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
248 """Get input and output history from the current session. Called by
256 """Get input and output history from the current session. Called by
249 get_range, and takes similar parameters."""
257 get_range, and takes similar parameters."""
250 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
258 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
251
259
252 n = len(input_hist)
260 n = len(input_hist)
253 if start < 0:
261 if start < 0:
254 start += n
262 start += n
255 if not stop:
263 if not stop:
256 stop = n
264 stop = n
257 elif stop < 0:
265 elif stop < 0:
258 stop += n
266 stop += n
259
267
260 for i in range(start, stop):
268 for i in range(start, stop):
261 if output:
269 if output:
262 line = (input_hist[i], self.output_hist_reprs.get(i))
270 line = (input_hist[i], self.output_hist_reprs.get(i))
263 else:
271 else:
264 line = input_hist[i]
272 line = input_hist[i]
265 yield (0, i, line)
273 yield (0, i, line)
266
274
267 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
275 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
268 """Retrieve input by session.
276 """Retrieve input by session.
269
277
270 Parameters
278 Parameters
271 ----------
279 ----------
272 session : int
280 session : int
273 Session number to retrieve. The current session is 0, and negative
281 Session number to retrieve. The current session is 0, and negative
274 numbers count back from current session, so -1 is previous session.
282 numbers count back from current session, so -1 is previous session.
275 start : int
283 start : int
276 First line to retrieve.
284 First line to retrieve.
277 stop : int
285 stop : int
278 End of line range (excluded from output itself). If None, retrieve
286 End of line range (excluded from output itself). If None, retrieve
279 to the end of the session.
287 to the end of the session.
280 raw : bool
288 raw : bool
281 If True, return untranslated input
289 If True, return untranslated input
282 output : bool
290 output : bool
283 If True, attempt to include output. This will be 'real' Python
291 If True, attempt to include output. This will be 'real' Python
284 objects for the current session, or text reprs from previous
292 objects for the current session, or text reprs from previous
285 sessions if db_log_output was enabled at the time. Where no output
293 sessions if db_log_output was enabled at the time. Where no output
286 is found, None is used.
294 is found, None is used.
287
295
288 Returns
296 Returns
289 -------
297 -------
290 An iterator over the desired lines. Each line is a 3-tuple, either
298 An iterator over the desired lines. Each line is a 3-tuple, either
291 (session, line, input) if output is False, or
299 (session, line, input) if output is False, or
292 (session, line, (input, output)) if output is True.
300 (session, line, (input, output)) if output is True.
293 """
301 """
294 if session == 0 or session==self.session_number: # Current session
302 if session == 0 or session==self.session_number: # Current session
295 return self._get_range_session(start, stop, raw, output)
303 return self._get_range_session(start, stop, raw, output)
296 if session < 0:
304 if session < 0:
297 session += self.session_number
305 session += self.session_number
298
306
299 if stop:
307 if stop:
300 lineclause = "line >= ? AND line < ?"
308 lineclause = "line >= ? AND line < ?"
301 params = (session, start, stop)
309 params = (session, start, stop)
302 else:
310 else:
303 lineclause = "line>=?"
311 lineclause = "line>=?"
304 params = (session, start)
312 params = (session, start)
305
313
306 return self._run_sql("WHERE session==? AND %s""" % lineclause,
314 return self._run_sql("WHERE session==? AND %s""" % lineclause,
307 params, raw=raw, output=output)
315 params, raw=raw, output=output)
308
316
309 def get_range_by_str(self, rangestr, raw=True, output=False):
317 def get_range_by_str(self, rangestr, raw=True, output=False):
310 """Get lines of history from a string of ranges, as used by magic
318 """Get lines of history from a string of ranges, as used by magic
311 commands %hist, %save, %macro, etc.
319 commands %hist, %save, %macro, etc.
312
320
313 Parameters
321 Parameters
314 ----------
322 ----------
315 rangestr : str
323 rangestr : str
316 A string specifying ranges, e.g. "5 ~2/1-4". See
324 A string specifying ranges, e.g. "5 ~2/1-4". See
317 :func:`magic_history` for full details.
325 :func:`magic_history` for full details.
318 raw, output : bool
326 raw, output : bool
319 As :meth:`get_range`
327 As :meth:`get_range`
320
328
321 Returns
329 Returns
322 -------
330 -------
323 Tuples as :meth:`get_range`
331 Tuples as :meth:`get_range`
324 """
332 """
325 for sess, s, e in extract_hist_ranges(rangestr):
333 for sess, s, e in extract_hist_ranges(rangestr):
326 for line in self.get_range(sess, s, e, raw=raw, output=output):
334 for line in self.get_range(sess, s, e, raw=raw, output=output):
327 yield line
335 yield line
328
336
329 ## ----------------------------
337 ## ----------------------------
330 ## Methods for storing history:
338 ## Methods for storing history:
331 ## ----------------------------
339 ## ----------------------------
332 def store_inputs(self, line_num, source, source_raw=None):
340 def store_inputs(self, line_num, source, source_raw=None):
333 """Store source and raw input in history and create input cache
341 """Store source and raw input in history and create input cache
334 variables _i*.
342 variables _i*.
335
343
336 Parameters
344 Parameters
337 ----------
345 ----------
338 line_num : int
346 line_num : int
339 The prompt number of this input.
347 The prompt number of this input.
340
348
341 source : str
349 source : str
342 Python input.
350 Python input.
343
351
344 source_raw : str, optional
352 source_raw : str, optional
345 If given, this is the raw input without any IPython transformations
353 If given, this is the raw input without any IPython transformations
346 applied to it. If not given, ``source`` is used.
354 applied to it. If not given, ``source`` is used.
347 """
355 """
348 if source_raw is None:
356 if source_raw is None:
349 source_raw = source
357 source_raw = source
350 source = source.rstrip('\n')
358 source = source.rstrip('\n')
351 source_raw = source_raw.rstrip('\n')
359 source_raw = source_raw.rstrip('\n')
352
360
353 # do not store exit/quit commands
361 # do not store exit/quit commands
354 if source_raw.strip() in self._exit_commands:
362 if source_raw.strip() in self._exit_commands:
355 return
363 return
356
364
357 self.input_hist_parsed.append(source)
365 self.input_hist_parsed.append(source)
358 self.input_hist_raw.append(source_raw)
366 self.input_hist_raw.append(source_raw)
359
367
360 self.db_input_cache.append((self.session_number, line_num,
368 self.db_input_cache.append((line_num, source, source_raw))
361 source, source_raw))
362 # Trigger to flush cache and write to DB.
369 # Trigger to flush cache and write to DB.
363 if len(self.db_input_cache) >= self.db_cache_size:
370 if len(self.db_input_cache) >= self.db_cache_size:
364 self.writeout_cache()
371 self.writeout_cache()
365
372
366 # update the auto _i variables
373 # update the auto _i variables
367 self._iii = self._ii
374 self._iii = self._ii
368 self._ii = self._i
375 self._ii = self._i
369 self._i = self._i00
376 self._i = self._i00
370 self._i00 = source_raw
377 self._i00 = source_raw
371
378
372 # hackish access to user namespace to create _i1,_i2... dynamically
379 # hackish access to user namespace to create _i1,_i2... dynamically
373 new_i = '_i%s' % line_num
380 new_i = '_i%s' % line_num
374 to_main = {'_i': self._i,
381 to_main = {'_i': self._i,
375 '_ii': self._ii,
382 '_ii': self._ii,
376 '_iii': self._iii,
383 '_iii': self._iii,
377 new_i : self._i00 }
384 new_i : self._i00 }
378 self.shell.user_ns.update(to_main)
385 self.shell.user_ns.update(to_main)
379
386
380 def store_output(self, line_num):
387 def store_output(self, line_num):
381 """If database output logging is enabled, this saves all the
388 """If database output logging is enabled, this saves all the
382 outputs from the indicated prompt number to the database. It's
389 outputs from the indicated prompt number to the database. It's
383 called by run_cell after code has been executed.
390 called by run_cell after code has been executed.
384
391
385 Parameters
392 Parameters
386 ----------
393 ----------
387 line_num : int
394 line_num : int
388 The line number from which to save outputs
395 The line number from which to save outputs
389 """
396 """
390 if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
397 if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
391 return
398 return
392 output = json.dumps(self.output_hist_reprs[line_num])
399 output = json.dumps(self.output_hist_reprs[line_num])
393 db_row = (self.session_number, line_num, output)
400
394 if self.db_cache_size > 1:
401 self.db_output_cache.append((line_num, output))
395 self.db_output_cache.append(db_row)
402 if self.db_cache_size <= 1:
396 else:
403 self.writeout_cache()
404
405 def _writeout_input_cache(self):
406 for line in self.db_input_cache:
397 with self.db:
407 with self.db:
398 self.db.execute("INSERT INTO output_history VALUES (?,?,?)", db_row)
408 self.db.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
409 (self.session_number,)+line)
399
410
400 def writeout_cache(self):
411 def _writeout_output_cache(self):
401 #print(self.db_input_cache)
412 for line in self.db_output_cache:
402 with self.db:
413 with self.db:
403 self.db.executemany("INSERT INTO history VALUES (?, ?, ?, ?)",
414 self.db.execute("INSERT INTO output_history VALUES (?, ?, ?)",
404 self.db_input_cache)
415 (self.session_number,)+line)
405 self.db.executemany("INSERT INTO output_history VALUES (?, ?, ?)",
416
406 self.db_output_cache)
417 def writeout_cache(self):
418 """Write any entries in the cache to the database."""
419 try:
420 self._writeout_input_cache()
421 except sqlite3.IntegrityError:
422 self.new_session()
423 print("ERROR! Session/line number was not unique in",
424 "database. History logging moved to new session",
425 self.session_number)
426 try: # Try writing to the new session. If this fails, don't recurse
427 self.writeout_cache()
428 except sqlite3.IntegrityError:
429 pass
430 finally:
407 self.db_input_cache = []
431 self.db_input_cache = []
432
433 try:
434 self._writeout_output_cache()
435 except sqlite3.IntegrityError:
436 print("!! Session/line number for output was not unique",
437 "in database. Output will not be stored.")
438 finally:
408 self.db_output_cache = []
439 self.db_output_cache = []
409
440
410
441
411 # To match, e.g. ~5/8-~2/3
442 # To match, e.g. ~5/8-~2/3
412 range_re = re.compile(r"""
443 range_re = re.compile(r"""
413 ((?P<startsess>~?\d+)/)?
444 ((?P<startsess>~?\d+)/)?
414 (?P<start>\d+) # Only the start line num is compulsory
445 (?P<start>\d+) # Only the start line num is compulsory
415 ((?P<sep>[\-:])
446 ((?P<sep>[\-:])
416 ((?P<endsess>~?\d+)/)?
447 ((?P<endsess>~?\d+)/)?
417 (?P<end>\d+))?
448 (?P<end>\d+))?
418 """, re.VERBOSE)
449 """, re.VERBOSE)
419
450
420 def extract_hist_ranges(ranges_str):
451 def extract_hist_ranges(ranges_str):
421 """Turn a string of history ranges into 3-tuples of (session, start, stop).
452 """Turn a string of history ranges into 3-tuples of (session, start, stop).
422
453
423 Examples
454 Examples
424 --------
455 --------
425 list(extract_input_ranges("~8/5-~7/4 2"))
456 list(extract_input_ranges("~8/5-~7/4 2"))
426 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
457 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
427 """
458 """
428 for range_str in ranges_str.split():
459 for range_str in ranges_str.split():
429 rmatch = range_re.match(range_str)
460 rmatch = range_re.match(range_str)
430 if not rmatch:
461 if not rmatch:
431 continue
462 continue
432 start = int(rmatch.group("start"))
463 start = int(rmatch.group("start"))
433 end = rmatch.group("end")
464 end = rmatch.group("end")
434 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
465 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
435 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
466 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
436 end += 1
467 end += 1
437 startsess = rmatch.group("startsess") or "0"
468 startsess = rmatch.group("startsess") or "0"
438 endsess = rmatch.group("endsess") or startsess
469 endsess = rmatch.group("endsess") or startsess
439 startsess = int(startsess.replace("~","-"))
470 startsess = int(startsess.replace("~","-"))
440 endsess = int(endsess.replace("~","-"))
471 endsess = int(endsess.replace("~","-"))
441 assert endsess >= startsess
472 assert endsess >= startsess
442
473
443 if endsess == startsess:
474 if endsess == startsess:
444 yield (startsess, start, end)
475 yield (startsess, start, end)
445 continue
476 continue
446 # Multiple sessions in one range:
477 # Multiple sessions in one range:
447 yield (startsess, start, None)
478 yield (startsess, start, None)
448 for sess in range(startsess+1, endsess):
479 for sess in range(startsess+1, endsess):
449 yield (sess, 1, None)
480 yield (sess, 1, None)
450 yield (endsess, 1, end)
481 yield (endsess, 1, end)
451
482
452 def _format_lineno(session, line):
483 def _format_lineno(session, line):
453 """Helper function to format line numbers properly."""
484 """Helper function to format line numbers properly."""
454 if session == 0:
485 if session == 0:
455 return str(line)
486 return str(line)
456 return "%s#%s" % (session, line)
487 return "%s#%s" % (session, line)
457
488
458 @testdec.skip_doctest
489 @testdec.skip_doctest
459 def magic_history(self, parameter_s = ''):
490 def magic_history(self, parameter_s = ''):
460 """Print input history (_i<n> variables), with most recent last.
491 """Print input history (_i<n> variables), with most recent last.
461
492
462 %history -> print at most 40 inputs (some may be multi-line)\\
493 %history -> print at most 40 inputs (some may be multi-line)\\
463 %history n -> print at most n inputs\\
494 %history n -> print at most n inputs\\
464 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
495 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
465
496
466 By default, input history is printed without line numbers so it can be
497 By default, input history is printed without line numbers so it can be
467 directly pasted into an editor. Use -n to show them.
498 directly pasted into an editor. Use -n to show them.
468
499
469 Ranges of history can be indicated using the syntax:
500 Ranges of history can be indicated using the syntax:
470 4 : Line 4, current session
501 4 : Line 4, current session
471 4-6 : Lines 4-6, current session
502 4-6 : Lines 4-6, current session
472 243/1-5: Lines 1-5, session 243
503 243/1-5: Lines 1-5, session 243
473 ~2/7 : Line 7, session 2 before current
504 ~2/7 : Line 7, session 2 before current
474 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
505 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
475 of 6 sessions ago.
506 of 6 sessions ago.
476 Multiple ranges can be entered, separated by spaces
507 Multiple ranges can be entered, separated by spaces
477
508
478 The same syntax is used by %macro, %save, %edit, %rerun
509 The same syntax is used by %macro, %save, %edit, %rerun
479
510
480 Options:
511 Options:
481
512
482 -n: print line numbers for each input.
513 -n: print line numbers for each input.
483 This feature is only available if numbered prompts are in use.
514 This feature is only available if numbered prompts are in use.
484
515
485 -o: also print outputs for each input.
516 -o: also print outputs for each input.
486
517
487 -p: print classic '>>>' python prompts before each input. This is useful
518 -p: print classic '>>>' python prompts before each input. This is useful
488 for making documentation, and in conjunction with -o, for producing
519 for making documentation, and in conjunction with -o, for producing
489 doctest-ready output.
520 doctest-ready output.
490
521
491 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
522 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
492
523
493 -t: print the 'translated' history, as IPython understands it. IPython
524 -t: print the 'translated' history, as IPython understands it. IPython
494 filters your input and converts it all into valid Python source before
525 filters your input and converts it all into valid Python source before
495 executing it (things like magics or aliases are turned into function
526 executing it (things like magics or aliases are turned into function
496 calls, for example). With this option, you'll see the native history
527 calls, for example). With this option, you'll see the native history
497 instead of the user-entered version: '%cd /' will be seen as
528 instead of the user-entered version: '%cd /' will be seen as
498 'get_ipython().magic("%cd /")' instead of '%cd /'.
529 'get_ipython().magic("%cd /")' instead of '%cd /'.
499
530
500 -g: treat the arg as a pattern to grep for in (full) history.
531 -g: treat the arg as a pattern to grep for in (full) history.
501 This includes the saved history (almost all commands ever written).
532 This includes the saved history (almost all commands ever written).
502 Use '%hist -g' to show full saved history (may be very long).
533 Use '%hist -g' to show full saved history (may be very long).
503
534
504 -l: get the last n lines from all sessions. Specify n as a single arg, or
535 -l: get the last n lines from all sessions. Specify n as a single arg, or
505 the default is the last 10 lines.
536 the default is the last 10 lines.
506
537
507 -f FILENAME: instead of printing the output to the screen, redirect it to
538 -f FILENAME: instead of printing the output to the screen, redirect it to
508 the given file. The file is always overwritten, though IPython asks for
539 the given file. The file is always overwritten, though IPython asks for
509 confirmation first if it already exists.
540 confirmation first if it already exists.
510
541
511 Examples
542 Examples
512 --------
543 --------
513 ::
544 ::
514
545
515 In [6]: %hist -n 4 6
546 In [6]: %hist -n 4 6
516 4:a = 12
547 4:a = 12
517 5:print a**2
548 5:print a**2
518
549
519 """
550 """
520
551
521 if not self.shell.displayhook.do_full_cache:
552 if not self.shell.displayhook.do_full_cache:
522 print('This feature is only available if numbered prompts are in use.')
553 print('This feature is only available if numbered prompts are in use.')
523 return
554 return
524 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
555 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
525
556
526 # For brevity
557 # For brevity
527 history_manager = self.shell.history_manager
558 history_manager = self.shell.history_manager
528
559
529 def _format_lineno(session, line):
560 def _format_lineno(session, line):
530 """Helper function to format line numbers properly."""
561 """Helper function to format line numbers properly."""
531 if session in (0, history_manager.session_number):
562 if session in (0, history_manager.session_number):
532 return str(line)
563 return str(line)
533 return "%s/%s" % (session, line)
564 return "%s/%s" % (session, line)
534
565
535 # Check if output to specific file was requested.
566 # Check if output to specific file was requested.
536 try:
567 try:
537 outfname = opts['f']
568 outfname = opts['f']
538 except KeyError:
569 except KeyError:
539 outfile = IPython.utils.io.Term.cout # default
570 outfile = IPython.utils.io.Term.cout # default
540 # We don't want to close stdout at the end!
571 # We don't want to close stdout at the end!
541 close_at_end = False
572 close_at_end = False
542 else:
573 else:
543 if os.path.exists(outfname):
574 if os.path.exists(outfname):
544 if not ask_yes_no("File %r exists. Overwrite?" % outfname):
575 if not ask_yes_no("File %r exists. Overwrite?" % outfname):
545 print('Aborting.')
576 print('Aborting.')
546 return
577 return
547
578
548 outfile = open(outfname,'w')
579 outfile = open(outfname,'w')
549 close_at_end = True
580 close_at_end = True
550
581
551 print_nums = 'n' in opts
582 print_nums = 'n' in opts
552 get_output = 'o' in opts
583 get_output = 'o' in opts
553 pyprompts = 'p' in opts
584 pyprompts = 'p' in opts
554 # Raw history is the default
585 # Raw history is the default
555 raw = not('t' in opts)
586 raw = not('t' in opts)
556
587
557 default_length = 40
588 default_length = 40
558 pattern = None
589 pattern = None
559
590
560 if 'g' in opts: # Glob search
591 if 'g' in opts: # Glob search
561 pattern = "*" + args + "*" if args else "*"
592 pattern = "*" + args + "*" if args else "*"
562 hist = history_manager.search(pattern, raw=raw, output=get_output)
593 hist = history_manager.search(pattern, raw=raw, output=get_output)
563 elif 'l' in opts: # Get 'tail'
594 elif 'l' in opts: # Get 'tail'
564 try:
595 try:
565 n = int(args)
596 n = int(args)
566 except ValueError, IndexError:
597 except ValueError, IndexError:
567 n = 10
598 n = 10
568 hist = history_manager.get_tail(n, raw=raw, output=get_output)
599 hist = history_manager.get_tail(n, raw=raw, output=get_output)
569 else:
600 else:
570 if args: # Get history by ranges
601 if args: # Get history by ranges
571 hist = history_manager.get_range_by_str(args, raw, get_output)
602 hist = history_manager.get_range_by_str(args, raw, get_output)
572 else: # Just get history for the current session
603 else: # Just get history for the current session
573 hist = history_manager.get_range(raw=raw, output=get_output)
604 hist = history_manager.get_range(raw=raw, output=get_output)
574
605
575 # We could be displaying the entire history, so let's not try to pull it
606 # We could be displaying the entire history, so let's not try to pull it
576 # into a list in memory. Anything that needs more space will just misalign.
607 # into a list in memory. Anything that needs more space will just misalign.
577 width = 4
608 width = 4
578
609
579 for session, lineno, inline in hist:
610 for session, lineno, inline in hist:
580 # Print user history with tabs expanded to 4 spaces. The GUI clients
611 # Print user history with tabs expanded to 4 spaces. The GUI clients
581 # use hard tabs for easier usability in auto-indented code, but we want
612 # use hard tabs for easier usability in auto-indented code, but we want
582 # to produce PEP-8 compliant history for safe pasting into an editor.
613 # to produce PEP-8 compliant history for safe pasting into an editor.
583 if get_output:
614 if get_output:
584 inline, output = inline
615 inline, output = inline
585 inline = inline.expandtabs(4).rstrip()
616 inline = inline.expandtabs(4).rstrip()
586
617
587 multiline = "\n" in inline
618 multiline = "\n" in inline
588 line_sep = '\n' if multiline else ' '
619 line_sep = '\n' if multiline else ' '
589 if print_nums:
620 if print_nums:
590 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
621 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
591 line_sep), file=outfile, end='')
622 line_sep), file=outfile, end='')
592 if pyprompts:
623 if pyprompts:
593 print(">>> ", end="", file=outfile)
624 print(">>> ", end="", file=outfile)
594 if multiline:
625 if multiline:
595 inline = "\n... ".join(inline.splitlines()) + "\n..."
626 inline = "\n... ".join(inline.splitlines()) + "\n..."
596 print(inline, file=outfile)
627 print(inline, file=outfile)
597 if get_output and output:
628 if get_output and output:
598 print("\n".join(output), file=outfile)
629 print("\n".join(output), file=outfile)
599
630
600 if close_at_end:
631 if close_at_end:
601 outfile.close()
632 outfile.close()
602
633
603
634
604 def magic_rep(self, arg):
635 def magic_rep(self, arg):
605 r""" Repeat a command, or get command to input line for editing
636 r""" Repeat a command, or get command to input line for editing
606
637
607 - %rep (no arguments):
638 - %rep (no arguments):
608
639
609 Place a string version of last computation result (stored in the special '_'
640 Place a string version of last computation result (stored in the special '_'
610 variable) to the next input prompt. Allows you to create elaborate command
641 variable) to the next input prompt. Allows you to create elaborate command
611 lines without using copy-paste::
642 lines without using copy-paste::
612
643
613 In[1]: l = ["hei", "vaan"]
644 In[1]: l = ["hei", "vaan"]
614 In[2]: "".join(l)
645 In[2]: "".join(l)
615 Out[2]: heivaan
646 Out[2]: heivaan
616 In[3]: %rep
647 In[3]: %rep
617 In[4]: heivaan_ <== cursor blinking
648 In[4]: heivaan_ <== cursor blinking
618
649
619 %rep 45
650 %rep 45
620
651
621 Place history line 45 on the next input prompt. Use %hist to find
652 Place history line 45 on the next input prompt. Use %hist to find
622 out the number.
653 out the number.
623
654
624 %rep 1-4
655 %rep 1-4
625
656
626 Combine the specified lines into one cell, and place it on the next
657 Combine the specified lines into one cell, and place it on the next
627 input prompt. See %history for the slice syntax.
658 input prompt. See %history for the slice syntax.
628
659
629 %rep foo+bar
660 %rep foo+bar
630
661
631 If foo+bar can be evaluated in the user namespace, the result is
662 If foo+bar can be evaluated in the user namespace, the result is
632 placed at the next input prompt. Otherwise, the history is searched
663 placed at the next input prompt. Otherwise, the history is searched
633 for lines which contain that substring, and the most recent one is
664 for lines which contain that substring, and the most recent one is
634 placed at the next input prompt.
665 placed at the next input prompt.
635 """
666 """
636 if not arg: # Last output
667 if not arg: # Last output
637 self.set_next_input(str(self.shell.user_ns["_"]))
668 self.set_next_input(str(self.shell.user_ns["_"]))
638 return
669 return
639 # Get history range
670 # Get history range
640 histlines = self.history_manager.get_range_by_str(arg)
671 histlines = self.history_manager.get_range_by_str(arg)
641 cmd = "\n".join(x[2] for x in histlines)
672 cmd = "\n".join(x[2] for x in histlines)
642 if cmd:
673 if cmd:
643 self.set_next_input(cmd.rstrip())
674 self.set_next_input(cmd.rstrip())
644 return
675 return
645
676
646 try: # Variable in user namespace
677 try: # Variable in user namespace
647 cmd = str(eval(arg, self.shell.user_ns))
678 cmd = str(eval(arg, self.shell.user_ns))
648 except Exception: # Search for term in history
679 except Exception: # Search for term in history
649 histlines = self.history_manager.search("*"+arg+"*")
680 histlines = self.history_manager.search("*"+arg+"*")
650 for h in reversed([x[2] for x in histlines]):
681 for h in reversed([x[2] for x in histlines]):
651 if 'rep' in h:
682 if 'rep' in h:
652 continue
683 continue
653 self.set_next_input(h.rstrip())
684 self.set_next_input(h.rstrip())
654 return
685 return
655 else:
686 else:
656 self.set_next_input(cmd.rstrip())
687 self.set_next_input(cmd.rstrip())
657 print("Couldn't evaluate or find in history:", arg)
688 print("Couldn't evaluate or find in history:", arg)
658
689
659 def magic_rerun(self, parameter_s=''):
690 def magic_rerun(self, parameter_s=''):
660 """Re-run previous input
691 """Re-run previous input
661
692
662 By default, you can specify ranges of input history to be repeated
693 By default, you can specify ranges of input history to be repeated
663 (as with %history). With no arguments, it will repeat the last line.
694 (as with %history). With no arguments, it will repeat the last line.
664
695
665 Options:
696 Options:
666
697
667 -l <n> : Repeat the last n lines of input, not including the
698 -l <n> : Repeat the last n lines of input, not including the
668 current command.
699 current command.
669
700
670 -g foo : Repeat the most recent line which contains foo
701 -g foo : Repeat the most recent line which contains foo
671 """
702 """
672 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
703 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
673 if "l" in opts: # Last n lines
704 if "l" in opts: # Last n lines
674 n = int(opts['l'])
705 n = int(opts['l'])
675 hist = self.history_manager.get_tail(n)
706 hist = self.history_manager.get_tail(n)
676 elif "g" in opts: # Search
707 elif "g" in opts: # Search
677 p = "*"+opts['g']+"*"
708 p = "*"+opts['g']+"*"
678 hist = list(self.history_manager.search(p))
709 hist = list(self.history_manager.search(p))
679 for l in reversed(hist):
710 for l in reversed(hist):
680 if "rerun" not in l[2]:
711 if "rerun" not in l[2]:
681 hist = [l] # The last match which isn't a %rerun
712 hist = [l] # The last match which isn't a %rerun
682 break
713 break
683 else:
714 else:
684 hist = [] # No matches except %rerun
715 hist = [] # No matches except %rerun
685 elif args: # Specify history ranges
716 elif args: # Specify history ranges
686 hist = self.history_manager.get_range_by_str(args)
717 hist = self.history_manager.get_range_by_str(args)
687 else: # Last line
718 else: # Last line
688 hist = self.history_manager.get_tail(1)
719 hist = self.history_manager.get_tail(1)
689 hist = [x[2] for x in hist]
720 hist = [x[2] for x in hist]
690 if not hist:
721 if not hist:
691 print("No lines in history match specification")
722 print("No lines in history match specification")
692 return
723 return
693 histlines = "\n".join(hist)
724 histlines = "\n".join(hist)
694 print("=== Executing: ===")
725 print("=== Executing: ===")
695 print(histlines)
726 print(histlines)
696 print("=== Output: ===")
727 print("=== Output: ===")
697 self.run_cell("\n".join(hist), store_history=False)
728 self.run_cell("\n".join(hist), store_history=False)
698
729
699
730
700 def init_ipython(ip):
731 def init_ipython(ip):
701 ip.define_magic("rep", magic_rep)
732 ip.define_magic("rep", magic_rep)
702 ip.define_magic("recall", magic_rep)
733 ip.define_magic("recall", magic_rep)
703 ip.define_magic("rerun", magic_rerun)
734 ip.define_magic("rerun", magic_rerun)
704 ip.define_magic("hist",magic_history) # Alternative name
735 ip.define_magic("hist",magic_history) # Alternative name
705 ip.define_magic("history",magic_history)
736 ip.define_magic("history",magic_history)
706
737
707 # XXX - ipy_completers are in quarantine, need to be updated to new apis
738 # XXX - ipy_completers are in quarantine, need to be updated to new apis
708 #import ipy_completers
739 #import ipy_completers
709 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
740 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
General Comments 0
You need to be logged in to leave comments. Login now