##// END OF EJS Templates
Simplify history retrieval code by moving query building and running into a common helper function.
Thomas Kluyver -
Show More
@@ -1,589 +1,596 b''
1 1 """ History related magics and functionality """
2 2 #-----------------------------------------------------------------------------
3 3 # Copyright (C) 2010 The IPython Development Team.
4 4 #
5 5 # Distributed under the terms of the BSD License.
6 6 #
7 7 # The full license is in the file COPYING.txt, distributed with this software.
8 8 #-----------------------------------------------------------------------------
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Imports
12 12 #-----------------------------------------------------------------------------
13 13 from __future__ import print_function
14 14
15 15 # Stdlib imports
16 16 import os
17 17 import re
18 18 import sqlite3
19 19
20 20 # Our own packages
21 21 from IPython.config.configurable import Configurable
22 22 import IPython.utils.io
23 23
24 24 from IPython.testing import decorators as testdec
25 25 from IPython.utils.io import ask_yes_no
26 26 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
27 27 from IPython.utils.warn import warn
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Classes and functions
31 31 #-----------------------------------------------------------------------------
32 32
33 33 class HistoryManager(Configurable):
34 34 """A class to organize all history-related functionality in one place.
35 35 """
36 36 # Public interface
37 37
38 38 # An instance of the IPython shell we are attached to
39 39 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
40 40 # Lists to hold processed and raw history. These start with a blank entry
41 41 # so that we can index them starting from 1
42 42 input_hist_parsed = List([""])
43 43 input_hist_raw = List([""])
44 44 # A list of directories visited during session
45 45 dir_hist = List()
46 46 # A dict of output history, keyed with ints from the shell's execution count
47 47 output_hist = Dict()
48 48 # String holding the path to the history file
49 49 hist_file = Unicode()
50 50 # The SQLite database
51 51 db = Instance(sqlite3.Connection)
52 52 # The number of the current session in the history database
53 53 session_number = Int()
54 54 # Should we log output to the database? (default no)
55 55 db_log_output = Bool(False, config=True)
56 56 # Write to database every x commands (higher values save disk access & power)
57 57 # Values of 1 or less effectively disable caching.
58 58 db_cache_size = Int(0, config=True)
59 59 # The input and output caches
60 60 db_input_cache = List()
61 61 db_output_cache = List()
62 62
63 63 # Private interface
64 64 # Variables used to store the three last inputs from the user. On each new
65 65 # history update, we populate the user's namespace with these, shifted as
66 66 # necessary.
67 67 _i00, _i, _ii, _iii = '','','',''
68 68
69 69 # A set with all forms of the exit command, so that we don't store them in
70 70 # the history (it's annoying to rewind the first entry and land on an exit
71 71 # call).
72 72 _exit_commands = None
73 73
74 74 def __init__(self, shell, config=None):
75 75 """Create a new history manager associated with a shell instance.
76 76 """
77 77 # We need a pointer back to the shell for various tasks.
78 78 super(HistoryManager, self).__init__(shell=shell, config=config)
79 79
80 80 # list of visited directories
81 81 try:
82 82 self.dir_hist = [os.getcwd()]
83 83 except OSError:
84 84 self.dir_hist = []
85 85
86 86 # Now the history file
87 87 if shell.profile:
88 88 histfname = 'history-%s' % shell.profile
89 89 else:
90 90 histfname = 'history'
91 91 self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
92 92 self.init_db()
93 93
94 94 self._i00, self._i, self._ii, self._iii = '','','',''
95 95
96 96 self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
97 97 '%quit', '%Exit', '%exit'])
98 98
99 99 def init_db(self):
100 100 """Connect to the database and get new session number."""
101 101 self.db = sqlite3.connect(self.hist_file)
102 102 self.db.execute("""CREATE TABLE IF NOT EXISTS history
103 103 (session integer, line integer, source text, source_raw text,
104 104 PRIMARY KEY (session, line))""")
105 105 # Output history is optional, but ensure the table's there so it can be
106 106 # enabled later.
107 107 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
108 108 (session integer, line integer, output text,
109 109 PRIMARY KEY (session, line))""")
110 110 cur = self.db.execute("""SELECT name FROM sqlite_master WHERE
111 111 type='table' AND name='singletons'""")
112 112 if not cur.fetchone():
113 113 self.db.execute("""CREATE TABLE singletons
114 114 (name text PRIMARY KEY, value)""")
115 115 self.db.execute("""INSERT INTO singletons VALUES
116 116 ('session_number', 1)""")
117 117 self.db.commit()
118 118 cur = self.db.execute("""SELECT value FROM singletons WHERE
119 119 name='session_number'""")
120 120 self.session_number = cur.fetchone()[0]
121 121
122 122 #Increment by one for next session.
123 123 self.db.execute("""UPDATE singletons SET value=? WHERE
124 124 name='session_number'""", (self.session_number+1,))
125 125 self.db.commit()
126 126
127 def get_hist_tail(self, n=10, raw=True, output=False):
128 """Get the last n lines from the history database."""
127 def _get_hist_sql(self, sql, params, raw=True, output=False):
128 """Prepares and runs an SQL query for the history database.
129
130 Parameters
131 ----------
132 sql : str
133 Any filtering expressions to go after SELECT ... FROM ...
134 params : tuple
135 Parameters passed to the SQL query (to replace "?")
136 raw : bool
137 If True, get raw input.
138 output :
139 If True, include output where available.
140
141 Returns
142 -------
143 An iterator over 3-tuples: (session, line_number, command), or if output
144 is True, (session, line_number, (command, output)).
145 """
129 146 toget = 'source_raw' if raw else 'source'
130 147 sqlfrom = "history"
131 148 if output:
132 149 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
133 150 toget = "history.%s, output_history.output" % toget
134 cur = self.db.execute("SELECT session, line, " + toget +\
135 " FROM "+sqlfrom+" ORDER BY session DESC, line DESC LIMIT ?", (n,))
136 hist = reversed(cur.fetchall())
137 if output:
138 return ((ses, lin, (inp, out)) for ses, lin, inp, out in hist)
139 return hist
151 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
152 (toget, sqlfrom) + sql, params)
153 if output: # Regroup into 3-tuples
154 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
155 return cur
156
157
158 def get_hist_tail(self, n=10, raw=True, output=False):
159 """Get the last n lines from the history database."""
160 cur = self._get_hist_sql("ORDER BY session DESC, line DESC LIMIT ?",
161 (n,), raw=raw, output=output)
162 return reversed(list(cur))
140 163
141 164 def get_hist_search(self, pattern="*", raw=True, output=False):
142 165 """Search the database using unix glob-style matching (wildcards * and
143 166 ?, escape using \).
144 167
145 168 Returns
146 169 -------
147 170 An iterator over tuples: (session, line_number, command)
148 171 """
149 toget = "source_raw" if raw else "source"
150 tosearch = toget
151 sqlfrom = "history"
172 tosearch = "source_raw" if raw else "source"
152 173 if output:
153 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
154 toget = "history.%s, output_history.output" % toget
155 174 tosearch = "history." + tosearch
156 hist = self.db.execute("SELECT session, line, " +toget+ \
157 " FROM "+sqlfrom+" WHERE " +tosearch+ " GLOB ?", (pattern,))
158 if output:
159 return ((ses, lin, (inp, out)) for ses, lin, inp, out in hist)
160 return hist
175 return self._get_hist_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
176 raw=raw, output=output)
161 177
162 178 def _get_hist_session(self, start=1, stop=None, raw=True, output=False):
163 179 """Get input and output history from the current session. Called by
164 180 get_history, and takes similar parameters."""
165 181 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
166 182
167 183 n = len(input_hist)
168 184 if start < 0:
169 185 start += n
170 186 if not stop:
171 187 stop = n
172 188 elif stop < 0:
173 189 stop += n
174 190
175 191 for i in range(start, stop):
176 192 if output:
177 193 line = (input_hist[i], repr(self.output_hist.get(i)))
178 194 else:
179 195 line = input_hist[i]
180 196 yield (0, i, line)
181 197
182 198 def get_history(self, session=0, start=1, stop=None, raw=True,output=False):
183 199 """Retrieve input by session.
184 200
185 201 Parameters
186 202 ----------
187 203 session : int
188 204 Session number to retrieve. The current session is 0, and negative
189 205 numbers count back from current session, so -1 is previous session.
190 206 start : int
191 207 First line to retrieve.
192 208 stop : int
193 209 End of line range (excluded from output itself). If None, retrieve
194 210 to the end of the session.
195 211 raw : bool
196 212 If True, return untranslated input
197 213 output : bool
198 214 If True, attempt to include output. This will be 'real' Python
199 215 objects for the current session, or text reprs from previous
200 216 sessions if db_log_output was enabled at the time. Where no output
201 217 is found, None is used.
202 218
203 219 Returns
204 220 -------
205 221 An iterator over the desired lines. Each line is a 3-tuple, either
206 222 (session, line, input) if output is False, or
207 223 (session, line, (input, output)) if output is True.
208 224 """
209 225 if session == 0 or session==self.session_number: # Current session
210 226 return self._get_hist_session(start, stop, raw, output)
211 227 if session < 0:
212 228 session += self.session_number
213 229
214 # Assemble the SQL query:
215 sqlfrom = "history"
216 toget = 'source_raw' if raw else 'source'
217 if output:
218 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
219 toget = "history.%s, output_history.output" % toget
220 230 if stop:
221 231 lineclause = "line >= ? AND line < ?"
222 232 params = (session, start, stop)
223 233 else:
224 234 lineclause = "line>=?"
225 235 params = (session, start)
226 236
227 cur = self.db.execute("""SELECT session, line, %s FROM %s WHERE
228 session==? AND %s""" %(toget, sqlfrom, lineclause), params)
229 if output: # Regroup into 3-tuples
230 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
231 return cur
237 return self._get_hist_sql("WHERE session==? AND %s""" % lineclause,
238 params, raw=raw, output=output)
232 239
233 240 def get_hist_from_rangestr(self, rangestr, raw=True, output=False):
234 241 """Get lines of history from a string of ranges, as used by magic
235 242 commands %hist, %save, %macro, etc."""
236 243 for sess, s, e in extract_hist_ranges(rangestr):
237 244 for line in self.get_history(sess, s, e, raw=raw, output=output):
238 245 yield line
239 246
240 247 def store_inputs(self, line_num, source, source_raw=None):
241 248 """Store source and raw input in history and create input cache
242 249 variables _i*.
243 250
244 251 Parameters
245 252 ----------
246 253 line_num : int
247 254 The prompt number of this input.
248 255
249 256 source : str
250 257 Python input.
251 258
252 259 source_raw : str, optional
253 260 If given, this is the raw input without any IPython transformations
254 261 applied to it. If not given, ``source`` is used.
255 262 """
256 263 if source_raw is None:
257 264 source_raw = source
258 265
259 266 # do not store exit/quit commands
260 267 if source_raw.strip() in self._exit_commands:
261 268 return
262 269
263 270 self.input_hist_parsed.append(source.rstrip())
264 271 self.input_hist_raw.append(source_raw.rstrip())
265 272
266 273 self.db_input_cache.append((self.session_number, line_num,
267 274 source, source_raw))
268 275 # Trigger to flush cache and write to DB.
269 276 if len(self.db_input_cache) >= self.db_cache_size:
270 277 self.writeout_cache()
271 278
272 279 # update the auto _i variables
273 280 self._iii = self._ii
274 281 self._ii = self._i
275 282 self._i = self._i00
276 283 self._i00 = source_raw
277 284
278 285 # hackish access to user namespace to create _i1,_i2... dynamically
279 286 new_i = '_i%s' % line_num
280 287 to_main = {'_i': self._i,
281 288 '_ii': self._ii,
282 289 '_iii': self._iii,
283 290 new_i : self._i00 }
284 291 self.shell.user_ns.update(to_main)
285 292
286 293 def store_output(self, line_num, output):
287 294 if not self.db_log_output:
288 295 return
289 296 db_row = (self.session_number, line_num, output)
290 297 if self.db_cache_size > 1:
291 298 self.db_output_cache.append(db_row)
292 299 else:
293 300 with self.db:
294 301 self.db.execute("INSERT INTO output_history VALUES (?,?,?)", db_row)
295 302
296 303 def writeout_cache(self):
297 304 #print(self.db_input_cache)
298 305 with self.db:
299 306 self.db.executemany("INSERT INTO history VALUES (?, ?, ?, ?)",
300 307 self.db_input_cache)
301 308 self.db.executemany("INSERT INTO output_history VALUES (?, ?, ?)",
302 309 self.db_output_cache)
303 310 self.db_input_cache = []
304 311 self.db_output_cache = []
305 312
306 313 def sync_inputs(self):
307 314 """Ensure raw and translated histories have same length."""
308 315 lr = len(self.input_hist_raw)
309 316 lp = len(self.input_hist_parsed)
310 317 if lp < lr:
311 318 self.input_hist_raw[:lr-lp] = []
312 319 elif lr < lp:
313 320 self.input_hist_parsed[:lp-lr] = []
314 321
315 322 def reset(self, new_session=True):
316 323 """Clear the current session's history, and (optionally) start a new
317 324 session."""
318 325 self.input_hist_parsed[:] = [""]
319 326 self.input_hist_raw[:] = [""]
320 327 self.output_hist.clear()
321 328 # The directory history can't be completely empty
322 329 self.dir_hist[:] = [os.getcwd()]
323 330
324 331 if new_session:
325 332 self.writeout_cache()
326 333 self.init_db() # Get new session number
327 334
328 335 # To match, e.g. ~5/8-~2/3
329 336 range_re = re.compile(r"""
330 337 ((?P<startsess>~?\d+)/)?
331 338 (?P<start>\d+) # Only the start line num is compulsory
332 339 ((?P<sep>[\-:])
333 340 ((?P<endsess>~?\d+)/)?
334 341 (?P<end>\d+))?
335 342 """, re.VERBOSE)
336 343
337 344 def extract_hist_ranges(ranges_str):
338 345 """Turn a string of history ranges into 3-tuples of (session, start, stop).
339 346
340 347 Examples
341 348 --------
342 349 list(extract_input_ranges("~8/5-~7/4 2"))
343 350 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
344 351 """
345 352 for range_str in ranges_str.split():
346 353 rmatch = range_re.match(range_str)
347 354 start = int(rmatch.group("start"))
348 355 end = rmatch.group("end")
349 356 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
350 357 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
351 358 end += 1
352 359 startsess = rmatch.group("startsess") or "0"
353 360 endsess = rmatch.group("endsess") or startsess
354 361 startsess = int(startsess.replace("~","-"))
355 362 endsess = int(endsess.replace("~","-"))
356 363 assert endsess >= startsess
357 364
358 365 if endsess == startsess:
359 366 yield (startsess, start, end)
360 367 continue
361 368 # Multiple sessions in one range:
362 369 yield (startsess, start, None)
363 370 for sess in range(startsess+1, endsess):
364 371 yield (sess, 1, None)
365 372 yield (endsess, 1, end)
366 373
367 374 def _format_lineno(session, line):
368 375 """Helper function to format line numbers properly."""
369 376 if session == 0:
370 377 return str(line)
371 378 return "%s#%s" % (session, line)
372 379
373 380 @testdec.skip_doctest
374 381 def magic_history(self, parameter_s = ''):
375 382 """Print input history (_i<n> variables), with most recent last.
376 383
377 384 %history -> print at most 40 inputs (some may be multi-line)\\
378 385 %history n -> print at most n inputs\\
379 386 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
380 387
381 388 By default, input history is printed without line numbers so it can be
382 389 directly pasted into an editor.
383 390
384 391 With -n, each input's number <n> is shown, and is accessible as the
385 392 automatically generated variable _i<n> as well as In[<n>]. Multi-line
386 393 statements are printed starting at a new line for easy copy/paste.
387 394
388 395 Options:
389 396
390 397 -n: print line numbers for each input.
391 398 This feature is only available if numbered prompts are in use.
392 399
393 400 -o: also print outputs for each input.
394 401
395 402 -p: print classic '>>>' python prompts before each input. This is useful
396 403 for making documentation, and in conjunction with -o, for producing
397 404 doctest-ready output.
398 405
399 406 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
400 407
401 408 -t: print the 'translated' history, as IPython understands it. IPython
402 409 filters your input and converts it all into valid Python source before
403 410 executing it (things like magics or aliases are turned into function
404 411 calls, for example). With this option, you'll see the native history
405 412 instead of the user-entered version: '%cd /' will be seen as
406 413 'get_ipython().magic("%cd /")' instead of '%cd /'.
407 414
408 415 -g: treat the arg as a pattern to grep for in (full) history.
409 416 This includes the saved history (almost all commands ever written).
410 417 Use '%hist -g' to show full saved history (may be very long).
411 418
412 419 -l: get the last n lines from all sessions. Specify n as a single arg, or
413 420 the default is the last 10 lines.
414 421
415 422 -f FILENAME: instead of printing the output to the screen, redirect it to
416 423 the given file. The file is always overwritten, though IPython asks for
417 424 confirmation first if it already exists.
418 425
419 426 Examples
420 427 --------
421 428 ::
422 429
423 430 In [6]: %hist -n 4 6
424 431 4:a = 12
425 432 5:print a**2
426 433
427 434 """
428 435
429 436 if not self.shell.displayhook.do_full_cache:
430 437 print('This feature is only available if numbered prompts are in use.')
431 438 return
432 439 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
433 440
434 441 # For brevity
435 442 history_manager = self.shell.history_manager
436 443
437 444 def _format_lineno(session, line):
438 445 """Helper function to format line numbers properly."""
439 446 if session in (0, history_manager.session_number):
440 447 return str(line)
441 448 return "%s/%s" % (session, line)
442 449
443 450 # Check if output to specific file was requested.
444 451 try:
445 452 outfname = opts['f']
446 453 except KeyError:
447 454 outfile = IPython.utils.io.Term.cout # default
448 455 # We don't want to close stdout at the end!
449 456 close_at_end = False
450 457 else:
451 458 if os.path.exists(outfname):
452 459 if not ask_yes_no("File %r exists. Overwrite?" % outfname):
453 460 print('Aborting.')
454 461 return
455 462
456 463 outfile = open(outfname,'w')
457 464 close_at_end = True
458 465
459 466 print_nums = 'n' in opts
460 467 get_output = 'o' in opts
461 468 pyprompts = 'p' in opts
462 469 # Raw history is the default
463 470 raw = not('t' in opts)
464 471
465 472 default_length = 40
466 473 pattern = None
467 474
468 475 if 'g' in opts: # Glob search
469 476 pattern = "*" + args + "*" if args else "*"
470 477 hist = history_manager.get_hist_search(pattern, raw=raw,
471 478 output=get_output)
472 479 elif 'l' in opts: # Get 'tail'
473 480 try:
474 481 n = int(args)
475 482 except ValueError, IndexError:
476 483 n = 10
477 484 hist = history_manager.get_hist_tail(n, raw=raw, output=get_output)
478 485 else:
479 486 if args: # Get history by ranges
480 487 hist = history_manager.get_hist_from_rangestr(args, raw, get_output)
481 488 else: # Just get history for the current session
482 489 hist = history_manager.get_history(raw=raw, output=get_output)
483 490
484 491 # We could be displaying the entire history, so let's not try to pull it
485 492 # into a list in memory. Anything that needs more space will just misalign.
486 493 width = 4
487 494
488 495 for session, lineno, inline in hist:
489 496 # Print user history with tabs expanded to 4 spaces. The GUI clients
490 497 # use hard tabs for easier usability in auto-indented code, but we want
491 498 # to produce PEP-8 compliant history for safe pasting into an editor.
492 499 if get_output:
493 500 inline, output = inline
494 501 inline = inline.expandtabs(4).rstrip()
495 502
496 503 multiline = "\n" in inline
497 504 line_sep = '\n' if multiline else ' '
498 505 if print_nums:
499 506 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
500 507 line_sep), file=outfile, end='')
501 508 if pyprompts:
502 509 print(">>> ", end="", file=outfile)
503 510 if multiline:
504 511 inline = "\n... ".join(inline.splitlines()) + "\n..."
505 512 print(inline, file=outfile)
506 513 if get_output and output:
507 514 print(output, file=outfile)
508 515
509 516 if close_at_end:
510 517 outfile.close()
511 518
512 519 # %hist is an alternative name
513 520 magic_hist = magic_history
514 521
515 522
516 523 def rep_f(self, arg):
517 524 r""" Repeat a command, or get command to input line for editing
518 525
519 526 - %rep (no arguments):
520 527
521 528 Place a string version of last computation result (stored in the special '_'
522 529 variable) to the next input prompt. Allows you to create elaborate command
523 530 lines without using copy-paste::
524 531
525 532 $ l = ["hei", "vaan"]
526 533 $ "".join(l)
527 534 ==> heivaan
528 535 $ %rep
529 536 $ heivaan_ <== cursor blinking
530 537
531 538 %rep 45
532 539
533 540 Place history line 45 to next input prompt. Use %hist to find out the
534 541 number.
535 542
536 543 %rep 1-4 6-7 3
537 544
538 545 Repeat the specified lines immediately. Input slice syntax is the same as
539 546 in %macro and %save.
540 547
541 548 %rep foo
542 549
543 550 Place the most recent line that has the substring "foo" to next input.
544 551 (e.g. 'svn ci -m foobar').
545 552 """
546 553
547 554 opts,args = self.parse_options(arg,'',mode='list')
548 555 if not args:
549 556 self.set_next_input(str(self.shell.user_ns["_"]))
550 557 return
551 558
552 559 if len(args) == 1 and not '-' in args[0]:
553 560 arg = args[0]
554 561 if len(arg) > 1 and arg.startswith('0'):
555 562 # get from shadow hist
556 563 num = int(arg[1:])
557 564 line = self.shell.shadowhist.get(num)
558 565 self.set_next_input(str(line))
559 566 return
560 567 try:
561 568 num = int(args[0])
562 569 self.set_next_input(str(self.shell.input_hist_raw[num]).rstrip())
563 570 return
564 571 except ValueError:
565 572 pass
566 573
567 574 for h in reversed(self.shell.input_hist_raw):
568 575 if 'rep' in h:
569 576 continue
570 577 if fnmatch.fnmatch(h,'*' + arg + '*'):
571 578 self.set_next_input(str(h).rstrip())
572 579 return
573 580
574 581 try:
575 582 lines = self.extract_input_slices(args, True)
576 583 print("lines", lines)
577 584 self.run_cell(lines)
578 585 except ValueError:
579 586 print("Not found in recent history:", args)
580 587
581 588
582 589 def init_ipython(ip):
583 590 ip.define_magic("rep",rep_f)
584 591 ip.define_magic("hist",magic_hist)
585 592 ip.define_magic("history",magic_history)
586 593
587 594 # XXX - ipy_completers are in quarantine, need to be updated to new apis
588 595 #import ipy_completers
589 596 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
General Comments 0
You need to be logged in to leave comments. Login now