##// END OF EJS Templates
Fix `history_manager.search(?, unique=True)` returning old entries
Nikita Kniazev -
Show More
@@ -1,907 +1,913
1 1 """ History related magics and functionality """
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6
7 7 import atexit
8 8 import datetime
9 9 from pathlib import Path
10 10 import re
11 11 import sqlite3
12 12 import threading
13 13
14 14 from traitlets.config.configurable import LoggingConfigurable
15 15 from decorator import decorator
16 16 from IPython.utils.decorators import undoc
17 17 from IPython.paths import locate_profile
18 18 from traitlets import (
19 19 Any,
20 20 Bool,
21 21 Dict,
22 22 Instance,
23 23 Integer,
24 24 List,
25 25 Unicode,
26 26 Union,
27 27 TraitError,
28 28 default,
29 29 observe,
30 30 )
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Classes and functions
34 34 #-----------------------------------------------------------------------------
35 35
36 36 @undoc
37 37 class DummyDB(object):
38 38 """Dummy DB that will act as a black hole for history.
39 39
40 40 Only used in the absence of sqlite"""
41 41 def execute(*args, **kwargs):
42 42 return []
43 43
44 44 def commit(self, *args, **kwargs):
45 45 pass
46 46
47 47 def __enter__(self, *args, **kwargs):
48 48 pass
49 49
50 50 def __exit__(self, *args, **kwargs):
51 51 pass
52 52
53 53
54 54 @decorator
55 55 def only_when_enabled(f, self, *a, **kw):
56 56 """Decorator: return an empty list in the absence of sqlite."""
57 57 if not self.enabled:
58 58 return []
59 59 else:
60 60 return f(self, *a, **kw)
61 61
62 62
63 63 # use 16kB as threshold for whether a corrupt history db should be saved
64 64 # that should be at least 100 entries or so
65 65 _SAVE_DB_SIZE = 16384
66 66
67 67 @decorator
68 68 def catch_corrupt_db(f, self, *a, **kw):
69 69 """A decorator which wraps HistoryAccessor method calls to catch errors from
70 70 a corrupt SQLite database, move the old database out of the way, and create
71 71 a new one.
72 72
73 73 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
74 74 not just a corrupt file.
75 75 """
76 76 try:
77 77 return f(self, *a, **kw)
78 78 except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
79 79 self._corrupt_db_counter += 1
80 80 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
81 81 if self.hist_file != ':memory:':
82 82 if self._corrupt_db_counter > self._corrupt_db_limit:
83 83 self.hist_file = ':memory:'
84 84 self.log.error("Failed to load history too many times, history will not be saved.")
85 85 elif self.hist_file.is_file():
86 86 # move the file out of the way
87 87 base = str(self.hist_file.parent / self.hist_file.stem)
88 88 ext = self.hist_file.suffix
89 89 size = self.hist_file.stat().st_size
90 90 if size >= _SAVE_DB_SIZE:
91 91 # if there's significant content, avoid clobbering
92 92 now = datetime.datetime.now().isoformat().replace(':', '.')
93 93 newpath = base + '-corrupt-' + now + ext
94 94 # don't clobber previous corrupt backups
95 95 for i in range(100):
96 96 if not Path(newpath).exists():
97 97 break
98 98 else:
99 99 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
100 100 else:
101 101 # not much content, possibly empty; don't worry about clobbering
102 102 # maybe we should just delete it?
103 103 newpath = base + '-corrupt' + ext
104 104 self.hist_file.rename(newpath)
105 105 self.log.error("History file was moved to %s and a new file created.", newpath)
106 106 self.init_db()
107 107 return []
108 108 else:
109 109 # Failed with :memory:, something serious is wrong
110 110 raise
111 111
112 112
113 113 class HistoryAccessorBase(LoggingConfigurable):
114 114 """An abstract class for History Accessors """
115 115
116 116 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
117 117 raise NotImplementedError
118 118
119 119 def search(self, pattern="*", raw=True, search_raw=True,
120 120 output=False, n=None, unique=False):
121 121 raise NotImplementedError
122 122
123 123 def get_range(self, session, start=1, stop=None, raw=True,output=False):
124 124 raise NotImplementedError
125 125
126 126 def get_range_by_str(self, rangestr, raw=True, output=False):
127 127 raise NotImplementedError
128 128
129 129
130 130 class HistoryAccessor(HistoryAccessorBase):
131 131 """Access the history database without adding to it.
132 132
133 133 This is intended for use by standalone history tools. IPython shells use
134 134 HistoryManager, below, which is a subclass of this."""
135 135
136 136 # counter for init_db retries, so we don't keep trying over and over
137 137 _corrupt_db_counter = 0
138 138 # after two failures, fallback on :memory:
139 139 _corrupt_db_limit = 2
140 140
141 141 # String holding the path to the history file
142 142 hist_file = Union(
143 143 [Instance(Path), Unicode()],
144 144 help="""Path to file to use for SQLite history database.
145 145
146 146 By default, IPython will put the history database in the IPython
147 147 profile directory. If you would rather share one history among
148 148 profiles, you can set this value in each, so that they are consistent.
149 149
150 150 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
151 151 mounts. If you see IPython hanging, try setting this to something on a
152 152 local disk, e.g::
153 153
154 154 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
155 155
156 156 you can also use the specific value `:memory:` (including the colon
157 157 at both end but not the back ticks), to avoid creating an history file.
158 158
159 159 """,
160 160 ).tag(config=True)
161 161
162 162 enabled = Bool(True,
163 163 help="""enable the SQLite history
164 164
165 165 set enabled=False to disable the SQLite history,
166 166 in which case there will be no stored history, no SQLite connection,
167 167 and no background saving thread. This may be necessary in some
168 168 threaded environments where IPython is embedded.
169 169 """
170 170 ).tag(config=True)
171 171
172 172 connection_options = Dict(
173 173 help="""Options for configuring the SQLite connection
174 174
175 175 These options are passed as keyword args to sqlite3.connect
176 176 when establishing database connections.
177 177 """
178 178 ).tag(config=True)
179 179
180 180 # The SQLite database
181 181 db = Any()
182 182 @observe('db')
183 183 def _db_changed(self, change):
184 184 """validate the db, since it can be an Instance of two different types"""
185 185 new = change['new']
186 186 connection_types = (DummyDB, sqlite3.Connection)
187 187 if not isinstance(new, connection_types):
188 188 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
189 189 (self.__class__.__name__, new)
190 190 raise TraitError(msg)
191 191
192 192 def __init__(self, profile="default", hist_file="", **traits):
193 193 """Create a new history accessor.
194 194
195 195 Parameters
196 196 ----------
197 197 profile : str
198 198 The name of the profile from which to open history.
199 199 hist_file : str
200 200 Path to an SQLite history database stored by IPython. If specified,
201 201 hist_file overrides profile.
202 202 config : :class:`~traitlets.config.loader.Config`
203 203 Config object. hist_file can also be set through this.
204 204 """
205 205 # We need a pointer back to the shell for various tasks.
206 206 super(HistoryAccessor, self).__init__(**traits)
207 207 # defer setting hist_file from kwarg until after init,
208 208 # otherwise the default kwarg value would clobber any value
209 209 # set by config
210 210 if hist_file:
211 211 self.hist_file = hist_file
212 212
213 213 try:
214 214 self.hist_file
215 215 except TraitError:
216 216 # No one has set the hist_file, yet.
217 217 self.hist_file = self._get_hist_file_name(profile)
218 218
219 219 self.init_db()
220 220
221 221 def _get_hist_file_name(self, profile='default'):
222 222 """Find the history file for the given profile name.
223 223
224 224 This is overridden by the HistoryManager subclass, to use the shell's
225 225 active profile.
226 226
227 227 Parameters
228 228 ----------
229 229 profile : str
230 230 The name of a profile which has a history file.
231 231 """
232 232 return Path(locate_profile(profile)) / "history.sqlite"
233 233
234 234 @catch_corrupt_db
235 235 def init_db(self):
236 236 """Connect to the database, and create tables if necessary."""
237 237 if not self.enabled:
238 238 self.db = DummyDB()
239 239 return
240 240
241 241 # use detect_types so that timestamps return datetime objects
242 242 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
243 243 kwargs.update(self.connection_options)
244 244 self.db = sqlite3.connect(str(self.hist_file), **kwargs)
245 245 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
246 246 primary key autoincrement, start timestamp,
247 247 end timestamp, num_cmds integer, remark text)""")
248 248 self.db.execute("""CREATE TABLE IF NOT EXISTS history
249 249 (session integer, line integer, source text, source_raw text,
250 250 PRIMARY KEY (session, line))""")
251 251 # Output history is optional, but ensure the table's there so it can be
252 252 # enabled later.
253 253 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
254 254 (session integer, line integer, output text,
255 255 PRIMARY KEY (session, line))""")
256 256 self.db.commit()
257 257 # success! reset corrupt db count
258 258 self._corrupt_db_counter = 0
259 259
260 260 def writeout_cache(self):
261 261 """Overridden by HistoryManager to dump the cache before certain
262 262 database lookups."""
263 263 pass
264 264
265 265 ## -------------------------------
266 266 ## Methods for retrieving history:
267 267 ## -------------------------------
268 def _run_sql(self, sql, params, raw=True, output=False):
268 def _run_sql(self, sql, params, raw=True, output=False, latest=False):
269 269 """Prepares and runs an SQL query for the history database.
270 270
271 271 Parameters
272 272 ----------
273 273 sql : str
274 274 Any filtering expressions to go after SELECT ... FROM ...
275 275 params : tuple
276 276 Parameters passed to the SQL query (to replace "?")
277 277 raw, output : bool
278 278 See :meth:`get_range`
279 latest : bool
280 Select rows with max (session, line)
279 281
280 282 Returns
281 283 -------
282 284 Tuples as :meth:`get_range`
283 285 """
284 286 toget = 'source_raw' if raw else 'source'
285 287 sqlfrom = "history"
286 288 if output:
287 289 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
288 290 toget = "history.%s, output_history.output" % toget
291 if latest:
292 toget += ", MAX(session * 128 * 1024 + line)"
289 293 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
290 294 (toget, sqlfrom) + sql, params)
295 if latest:
296 cur = (row[:-1] for row in cur)
291 297 if output: # Regroup into 3-tuples, and parse JSON
292 298 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
293 299 return cur
294 300
295 301 @only_when_enabled
296 302 @catch_corrupt_db
297 303 def get_session_info(self, session):
298 304 """Get info about a session.
299 305
300 306 Parameters
301 307 ----------
302 308
303 309 session : int
304 310 Session number to retrieve.
305 311
306 312 Returns
307 313 -------
308 314
309 315 session_id : int
310 316 Session ID number
311 317 start : datetime
312 318 Timestamp for the start of the session.
313 319 end : datetime
314 320 Timestamp for the end of the session, or None if IPython crashed.
315 321 num_cmds : int
316 322 Number of commands run, or None if IPython crashed.
317 323 remark : unicode
318 324 A manually set description.
319 325 """
320 326 query = "SELECT * from sessions where session == ?"
321 327 return self.db.execute(query, (session,)).fetchone()
322 328
323 329 @catch_corrupt_db
324 330 def get_last_session_id(self):
325 331 """Get the last session ID currently in the database.
326 332
327 333 Within IPython, this should be the same as the value stored in
328 334 :attr:`HistoryManager.session_number`.
329 335 """
330 336 for record in self.get_tail(n=1, include_latest=True):
331 337 return record[0]
332 338
333 339 @catch_corrupt_db
334 340 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
335 341 """Get the last n lines from the history database.
336 342
337 343 Parameters
338 344 ----------
339 345 n : int
340 346 The number of lines to get
341 347 raw, output : bool
342 348 See :meth:`get_range`
343 349 include_latest : bool
344 350 If False (default), n+1 lines are fetched, and the latest one
345 351 is discarded. This is intended to be used where the function
346 352 is called by a user command, which it should not return.
347 353
348 354 Returns
349 355 -------
350 356 Tuples as :meth:`get_range`
351 357 """
352 358 self.writeout_cache()
353 359 if not include_latest:
354 360 n += 1
355 361 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
356 362 (n,), raw=raw, output=output)
357 363 if not include_latest:
358 364 return reversed(list(cur)[1:])
359 365 return reversed(list(cur))
360 366
361 367 @catch_corrupt_db
362 368 def search(self, pattern="*", raw=True, search_raw=True,
363 369 output=False, n=None, unique=False):
364 370 """Search the database using unix glob-style matching (wildcards
365 371 * and ?).
366 372
367 373 Parameters
368 374 ----------
369 375 pattern : str
370 376 The wildcarded pattern to match when searching
371 377 search_raw : bool
372 378 If True, search the raw input, otherwise, the parsed input
373 379 raw, output : bool
374 380 See :meth:`get_range`
375 381 n : None or int
376 382 If an integer is given, it defines the limit of
377 383 returned entries.
378 384 unique : bool
379 385 When it is true, return only unique entries.
380 386
381 387 Returns
382 388 -------
383 389 Tuples as :meth:`get_range`
384 390 """
385 391 tosearch = "source_raw" if search_raw else "source"
386 392 if output:
387 393 tosearch = "history." + tosearch
388 394 self.writeout_cache()
389 395 sqlform = "WHERE %s GLOB ?" % tosearch
390 396 params = (pattern,)
391 397 if unique:
392 398 sqlform += ' GROUP BY {0}'.format(tosearch)
393 399 if n is not None:
394 400 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
395 401 params += (n,)
396 402 elif unique:
397 403 sqlform += " ORDER BY session, line"
398 cur = self._run_sql(sqlform, params, raw=raw, output=output)
404 cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
399 405 if n is not None:
400 406 return reversed(list(cur))
401 407 return cur
402 408
403 409 @catch_corrupt_db
404 410 def get_range(self, session, start=1, stop=None, raw=True,output=False):
405 411 """Retrieve input by session.
406 412
407 413 Parameters
408 414 ----------
409 415 session : int
410 416 Session number to retrieve.
411 417 start : int
412 418 First line to retrieve.
413 419 stop : int
414 420 End of line range (excluded from output itself). If None, retrieve
415 421 to the end of the session.
416 422 raw : bool
417 423 If True, return untranslated input
418 424 output : bool
419 425 If True, attempt to include output. This will be 'real' Python
420 426 objects for the current session, or text reprs from previous
421 427 sessions if db_log_output was enabled at the time. Where no output
422 428 is found, None is used.
423 429
424 430 Returns
425 431 -------
426 432 entries
427 433 An iterator over the desired lines. Each line is a 3-tuple, either
428 434 (session, line, input) if output is False, or
429 435 (session, line, (input, output)) if output is True.
430 436 """
431 437 if stop:
432 438 lineclause = "line >= ? AND line < ?"
433 439 params = (session, start, stop)
434 440 else:
435 441 lineclause = "line>=?"
436 442 params = (session, start)
437 443
438 444 return self._run_sql("WHERE session==? AND %s" % lineclause,
439 445 params, raw=raw, output=output)
440 446
441 447 def get_range_by_str(self, rangestr, raw=True, output=False):
442 448 """Get lines of history from a string of ranges, as used by magic
443 449 commands %hist, %save, %macro, etc.
444 450
445 451 Parameters
446 452 ----------
447 453 rangestr : str
448 454 A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
449 455 this will return everything from current session's history.
450 456
451 457 See the documentation of :func:`%history` for the full details.
452 458
453 459 raw, output : bool
454 460 As :meth:`get_range`
455 461
456 462 Returns
457 463 -------
458 464 Tuples as :meth:`get_range`
459 465 """
460 466 for sess, s, e in extract_hist_ranges(rangestr):
461 467 for line in self.get_range(sess, s, e, raw=raw, output=output):
462 468 yield line
463 469
464 470
465 471 class HistoryManager(HistoryAccessor):
466 472 """A class to organize all history-related functionality in one place.
467 473 """
468 474 # Public interface
469 475
470 476 # An instance of the IPython shell we are attached to
471 477 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
472 478 allow_none=True)
473 479 # Lists to hold processed and raw history. These start with a blank entry
474 480 # so that we can index them starting from 1
475 481 input_hist_parsed = List([""])
476 482 input_hist_raw = List([""])
477 483 # A list of directories visited during session
478 484 dir_hist = List()
479 485 @default('dir_hist')
480 486 def _dir_hist_default(self):
481 487 try:
482 488 return [Path.cwd()]
483 489 except OSError:
484 490 return []
485 491
486 492 # A dict of output history, keyed with ints from the shell's
487 493 # execution count.
488 494 output_hist = Dict()
489 495 # The text/plain repr of outputs.
490 496 output_hist_reprs = Dict()
491 497
492 498 # The number of the current session in the history database
493 499 session_number = Integer()
494 500
495 501 db_log_output = Bool(False,
496 502 help="Should the history database include output? (default: no)"
497 503 ).tag(config=True)
498 504 db_cache_size = Integer(0,
499 505 help="Write to database every x commands (higher values save disk access & power).\n"
500 506 "Values of 1 or less effectively disable caching."
501 507 ).tag(config=True)
502 508 # The input and output caches
503 509 db_input_cache = List()
504 510 db_output_cache = List()
505 511
506 512 # History saving in separate thread
507 513 save_thread = Instance('IPython.core.history.HistorySavingThread',
508 514 allow_none=True)
509 515 save_flag = Instance(threading.Event, allow_none=True)
510 516
511 517 # Private interface
512 518 # Variables used to store the three last inputs from the user. On each new
513 519 # history update, we populate the user's namespace with these, shifted as
514 520 # necessary.
515 521 _i00 = Unicode(u'')
516 522 _i = Unicode(u'')
517 523 _ii = Unicode(u'')
518 524 _iii = Unicode(u'')
519 525
520 526 # A regex matching all forms of the exit command, so that we don't store
521 527 # them in the history (it's annoying to rewind the first entry and land on
522 528 # an exit call).
523 529 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
524 530
525 531 def __init__(self, shell=None, config=None, **traits):
526 532 """Create a new history manager associated with a shell instance.
527 533 """
528 534 # We need a pointer back to the shell for various tasks.
529 535 super(HistoryManager, self).__init__(shell=shell, config=config,
530 536 **traits)
531 537 self.save_flag = threading.Event()
532 538 self.db_input_cache_lock = threading.Lock()
533 539 self.db_output_cache_lock = threading.Lock()
534 540
535 541 try:
536 542 self.new_session()
537 543 except sqlite3.OperationalError:
538 544 self.log.error("Failed to create history session in %s. History will not be saved.",
539 545 self.hist_file, exc_info=True)
540 546 self.hist_file = ':memory:'
541 547
542 548 if self.enabled and self.hist_file != ':memory:':
543 549 self.save_thread = HistorySavingThread(self)
544 550 self.save_thread.start()
545 551
546 552 def _get_hist_file_name(self, profile=None):
547 553 """Get default history file name based on the Shell's profile.
548 554
549 555 The profile parameter is ignored, but must exist for compatibility with
550 556 the parent class."""
551 557 profile_dir = self.shell.profile_dir.location
552 558 return Path(profile_dir) / "history.sqlite"
553 559
554 560 @only_when_enabled
555 561 def new_session(self, conn=None):
556 562 """Get a new session number."""
557 563 if conn is None:
558 564 conn = self.db
559 565
560 566 with conn:
561 567 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
562 568 NULL, "") """, (datetime.datetime.now(),))
563 569 self.session_number = cur.lastrowid
564 570
565 571 def end_session(self):
566 572 """Close the database session, filling in the end time and line count."""
567 573 self.writeout_cache()
568 574 with self.db:
569 575 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
570 576 session==?""", (datetime.datetime.now(),
571 577 len(self.input_hist_parsed)-1, self.session_number))
572 578 self.session_number = 0
573 579
574 580 def name_session(self, name):
575 581 """Give the current session a name in the history database."""
576 582 with self.db:
577 583 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
578 584 (name, self.session_number))
579 585
580 586 def reset(self, new_session=True):
581 587 """Clear the session history, releasing all object references, and
582 588 optionally open a new session."""
583 589 self.output_hist.clear()
584 590 # The directory history can't be completely empty
585 591 self.dir_hist[:] = [Path.cwd()]
586 592
587 593 if new_session:
588 594 if self.session_number:
589 595 self.end_session()
590 596 self.input_hist_parsed[:] = [""]
591 597 self.input_hist_raw[:] = [""]
592 598 self.new_session()
593 599
594 600 # ------------------------------
595 601 # Methods for retrieving history
596 602 # ------------------------------
597 603 def get_session_info(self, session=0):
598 604 """Get info about a session.
599 605
600 606 Parameters
601 607 ----------
602 608
603 609 session : int
604 610 Session number to retrieve. The current session is 0, and negative
605 611 numbers count back from current session, so -1 is the previous session.
606 612
607 613 Returns
608 614 -------
609 615
610 616 session_id : int
611 617 Session ID number
612 618 start : datetime
613 619 Timestamp for the start of the session.
614 620 end : datetime
615 621 Timestamp for the end of the session, or None if IPython crashed.
616 622 num_cmds : int
617 623 Number of commands run, or None if IPython crashed.
618 624 remark : unicode
619 625 A manually set description.
620 626 """
621 627 if session <= 0:
622 628 session += self.session_number
623 629
624 630 return super(HistoryManager, self).get_session_info(session=session)
625 631
626 632 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
627 633 """Get input and output history from the current session. Called by
628 634 get_range, and takes similar parameters."""
629 635 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
630 636
631 637 n = len(input_hist)
632 638 if start < 0:
633 639 start += n
634 640 if not stop or (stop > n):
635 641 stop = n
636 642 elif stop < 0:
637 643 stop += n
638 644
639 645 for i in range(start, stop):
640 646 if output:
641 647 line = (input_hist[i], self.output_hist_reprs.get(i))
642 648 else:
643 649 line = input_hist[i]
644 650 yield (0, i, line)
645 651
646 652 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
647 653 """Retrieve input by session.
648 654
649 655 Parameters
650 656 ----------
651 657 session : int
652 658 Session number to retrieve. The current session is 0, and negative
653 659 numbers count back from current session, so -1 is previous session.
654 660 start : int
655 661 First line to retrieve.
656 662 stop : int
657 663 End of line range (excluded from output itself). If None, retrieve
658 664 to the end of the session.
659 665 raw : bool
660 666 If True, return untranslated input
661 667 output : bool
662 668 If True, attempt to include output. This will be 'real' Python
663 669 objects for the current session, or text reprs from previous
664 670 sessions if db_log_output was enabled at the time. Where no output
665 671 is found, None is used.
666 672
667 673 Returns
668 674 -------
669 675 entries
670 676 An iterator over the desired lines. Each line is a 3-tuple, either
671 677 (session, line, input) if output is False, or
672 678 (session, line, (input, output)) if output is True.
673 679 """
674 680 if session <= 0:
675 681 session += self.session_number
676 682 if session==self.session_number: # Current session
677 683 return self._get_range_session(start, stop, raw, output)
678 684 return super(HistoryManager, self).get_range(session, start, stop, raw,
679 685 output)
680 686
681 687 ## ----------------------------
682 688 ## Methods for storing history:
683 689 ## ----------------------------
684 690 def store_inputs(self, line_num, source, source_raw=None):
685 691 """Store source and raw input in history and create input cache
686 692 variables ``_i*``.
687 693
688 694 Parameters
689 695 ----------
690 696 line_num : int
691 697 The prompt number of this input.
692 698
693 699 source : str
694 700 Python input.
695 701
696 702 source_raw : str, optional
697 703 If given, this is the raw input without any IPython transformations
698 704 applied to it. If not given, ``source`` is used.
699 705 """
700 706 if source_raw is None:
701 707 source_raw = source
702 708 source = source.rstrip('\n')
703 709 source_raw = source_raw.rstrip('\n')
704 710
705 711 # do not store exit/quit commands
706 712 if self._exit_re.match(source_raw.strip()):
707 713 return
708 714
709 715 self.input_hist_parsed.append(source)
710 716 self.input_hist_raw.append(source_raw)
711 717
712 718 with self.db_input_cache_lock:
713 719 self.db_input_cache.append((line_num, source, source_raw))
714 720 # Trigger to flush cache and write to DB.
715 721 if len(self.db_input_cache) >= self.db_cache_size:
716 722 self.save_flag.set()
717 723
718 724 # update the auto _i variables
719 725 self._iii = self._ii
720 726 self._ii = self._i
721 727 self._i = self._i00
722 728 self._i00 = source_raw
723 729
724 730 # hackish access to user namespace to create _i1,_i2... dynamically
725 731 new_i = '_i%s' % line_num
726 732 to_main = {'_i': self._i,
727 733 '_ii': self._ii,
728 734 '_iii': self._iii,
729 735 new_i : self._i00 }
730 736
731 737 if self.shell is not None:
732 738 self.shell.push(to_main, interactive=False)
733 739
734 740 def store_output(self, line_num):
735 741 """If database output logging is enabled, this saves all the
736 742 outputs from the indicated prompt number to the database. It's
737 743 called by run_cell after code has been executed.
738 744
739 745 Parameters
740 746 ----------
741 747 line_num : int
742 748 The line number from which to save outputs
743 749 """
744 750 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
745 751 return
746 752 output = self.output_hist_reprs[line_num]
747 753
748 754 with self.db_output_cache_lock:
749 755 self.db_output_cache.append((line_num, output))
750 756 if self.db_cache_size <= 1:
751 757 self.save_flag.set()
752 758
753 759 def _writeout_input_cache(self, conn):
754 760 with conn:
755 761 for line in self.db_input_cache:
756 762 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
757 763 (self.session_number,)+line)
758 764
759 765 def _writeout_output_cache(self, conn):
760 766 with conn:
761 767 for line in self.db_output_cache:
762 768 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
763 769 (self.session_number,)+line)
764 770
765 771 @only_when_enabled
766 772 def writeout_cache(self, conn=None):
767 773 """Write any entries in the cache to the database."""
768 774 if conn is None:
769 775 conn = self.db
770 776
771 777 with self.db_input_cache_lock:
772 778 try:
773 779 self._writeout_input_cache(conn)
774 780 except sqlite3.IntegrityError:
775 781 self.new_session(conn)
776 782 print("ERROR! Session/line number was not unique in",
777 783 "database. History logging moved to new session",
778 784 self.session_number)
779 785 try:
780 786 # Try writing to the new session. If this fails, don't
781 787 # recurse
782 788 self._writeout_input_cache(conn)
783 789 except sqlite3.IntegrityError:
784 790 pass
785 791 finally:
786 792 self.db_input_cache = []
787 793
788 794 with self.db_output_cache_lock:
789 795 try:
790 796 self._writeout_output_cache(conn)
791 797 except sqlite3.IntegrityError:
792 798 print("!! Session/line number for output was not unique",
793 799 "in database. Output will not be stored.")
794 800 finally:
795 801 self.db_output_cache = []
796 802
797 803
798 804 class HistorySavingThread(threading.Thread):
799 805 """This thread takes care of writing history to the database, so that
800 806 the UI isn't held up while that happens.
801 807
802 808 It waits for the HistoryManager's save_flag to be set, then writes out
803 809 the history cache. The main thread is responsible for setting the flag when
804 810 the cache size reaches a defined threshold."""
805 811 daemon = True
806 812 stop_now = False
807 813 enabled = True
808 814 def __init__(self, history_manager):
809 815 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
810 816 self.history_manager = history_manager
811 817 self.enabled = history_manager.enabled
812 818 atexit.register(self.stop)
813 819
814 820 @only_when_enabled
815 821 def run(self):
816 822 # We need a separate db connection per thread:
817 823 try:
818 824 self.db = sqlite3.connect(
819 825 str(self.history_manager.hist_file),
820 **self.history_manager.connection_options
826 **self.history_manager.connection_options,
821 827 )
822 828 while True:
823 829 self.history_manager.save_flag.wait()
824 830 if self.stop_now:
825 831 self.db.close()
826 832 return
827 833 self.history_manager.save_flag.clear()
828 834 self.history_manager.writeout_cache(self.db)
829 835 except Exception as e:
830 836 print(("The history saving thread hit an unexpected error (%s)."
831 837 "History will not be written to the database.") % repr(e))
832 838
833 839 def stop(self):
834 840 """This can be called from the main thread to safely stop this thread.
835 841
836 842 Note that it does not attempt to write out remaining history before
837 843 exiting. That should be done by calling the HistoryManager's
838 844 end_session method."""
839 845 self.stop_now = True
840 846 self.history_manager.save_flag.set()
841 847 self.join()
842 848
843 849
844 850 # To match, e.g. ~5/8-~2/3
845 851 range_re = re.compile(r"""
846 852 ((?P<startsess>~?\d+)/)?
847 853 (?P<start>\d+)?
848 854 ((?P<sep>[\-:])
849 855 ((?P<endsess>~?\d+)/)?
850 856 (?P<end>\d+))?
851 857 $""", re.VERBOSE)
852 858
853 859
854 860 def extract_hist_ranges(ranges_str):
855 861 """Turn a string of history ranges into 3-tuples of (session, start, stop).
856 862
857 863 Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
858 864 session".
859 865
860 866 Examples
861 867 --------
862 868 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
863 869 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
864 870 """
865 871 if ranges_str == "":
866 872 yield (0, 1, None) # Everything from current session
867 873 return
868 874
869 875 for range_str in ranges_str.split():
870 876 rmatch = range_re.match(range_str)
871 877 if not rmatch:
872 878 continue
873 879 start = rmatch.group("start")
874 880 if start:
875 881 start = int(start)
876 882 end = rmatch.group("end")
877 883 # If no end specified, get (a, a + 1)
878 884 end = int(end) if end else start + 1
879 885 else: # start not specified
880 886 if not rmatch.group('startsess'): # no startsess
881 887 continue
882 888 start = 1
883 889 end = None # provide the entire session hist
884 890
885 891 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
886 892 end += 1
887 893 startsess = rmatch.group("startsess") or "0"
888 894 endsess = rmatch.group("endsess") or startsess
889 895 startsess = int(startsess.replace("~","-"))
890 896 endsess = int(endsess.replace("~","-"))
891 897 assert endsess >= startsess, "start session must be earlier than end session"
892 898
893 899 if endsess == startsess:
894 900 yield (startsess, start, end)
895 901 continue
896 902 # Multiple sessions in one range:
897 903 yield (startsess, start, None)
898 904 for sess in range(startsess+1, endsess):
899 905 yield (sess, 1, None)
900 906 yield (endsess, 1, end)
901 907
902 908
903 909 def _format_lineno(session, line):
904 910 """Helper function to format line numbers properly."""
905 911 if session == 0:
906 912 return str(line)
907 913 return "%s#%s" % (session, line)
@@ -1,218 +1,227
1 1 # coding: utf-8
2 2 """Tests for the IPython tab-completion machinery.
3 3 """
4 4 #-----------------------------------------------------------------------------
5 5 # Module imports
6 6 #-----------------------------------------------------------------------------
7 7
8 8 # stdlib
9 9 import io
10 10 from pathlib import Path
11 11 import sys
12 12 import tempfile
13 13 from datetime import datetime
14 14 import sqlite3
15 15
16 16 # our own packages
17 17 from traitlets.config.loader import Config
18 18 from IPython.utils.tempdir import TemporaryDirectory
19 19 from IPython.core.history import HistoryManager, extract_hist_ranges
20 from IPython.testing.decorators import skipif
21 20
22 21 def test_proper_default_encoding():
23 22 assert sys.getdefaultencoding() == "utf-8"
24 23
25 @skipif(sqlite3.sqlite_version_info > (3,24,0))
26 24 def test_history():
27 25 ip = get_ipython()
28 26 with TemporaryDirectory() as tmpdir:
29 27 tmp_path = Path(tmpdir)
30 28 hist_manager_ori = ip.history_manager
31 29 hist_file = tmp_path / "history.sqlite"
32 30 try:
33 31 ip.history_manager = HistoryManager(shell=ip, hist_file=hist_file)
34 32 hist = ["a=1", "def f():\n test = 1\n return test", "b='€Æ¾÷ß'"]
35 33 for i, h in enumerate(hist, start=1):
36 34 ip.history_manager.store_inputs(i, h)
37 35
38 36 ip.history_manager.db_log_output = True
39 37 # Doesn't match the input, but we'll just check it's stored.
40 38 ip.history_manager.output_hist_reprs[3] = "spam"
41 39 ip.history_manager.store_output(3)
42 40
43 41 assert ip.history_manager.input_hist_raw == [""] + hist
44 42
45 43 # Detailed tests for _get_range_session
46 44 grs = ip.history_manager._get_range_session
47 45 assert list(grs(start=2, stop=-1)) == list(zip([0], [2], hist[1:-1]))
48 46 assert list(grs(start=-2)) == list(zip([0, 0], [2, 3], hist[-2:]))
49 47 assert list(grs(output=True)) == list(
50 48 zip([0, 0, 0], [1, 2, 3], zip(hist, [None, None, "spam"]))
51 49 )
52 50
53 51 # Check whether specifying a range beyond the end of the current
54 52 # session results in an error (gh-804)
55 53 ip.magic('%hist 2-500')
56 54
57 55 # Check that we can write non-ascii characters to a file
58 56 ip.magic("%%hist -f %s" % (tmp_path / "test1"))
59 57 ip.magic("%%hist -pf %s" % (tmp_path / "test2"))
60 58 ip.magic("%%hist -nf %s" % (tmp_path / "test3"))
61 59 ip.magic("%%save %s 1-10" % (tmp_path / "test4"))
62 60
63 61 # New session
64 62 ip.history_manager.reset()
65 63 newcmds = ["z=5", "class X(object):\n pass", "k='p'", "z=5"]
66 64 for i, cmd in enumerate(newcmds, start=1):
67 65 ip.history_manager.store_inputs(i, cmd)
68 66 gothist = ip.history_manager.get_range(start=1, stop=4)
69 67 assert list(gothist) == list(zip([0, 0, 0], [1, 2, 3], newcmds))
70 68 # Previous session:
71 69 gothist = ip.history_manager.get_range(-1, 1, 4)
72 70 assert list(gothist) == list(zip([1, 1, 1], [1, 2, 3], hist))
73 71
74 72 newhist = [(2, i, c) for (i, c) in enumerate(newcmds, 1)]
75 73
76 74 # Check get_hist_tail
77 75 gothist = ip.history_manager.get_tail(5, output=True,
78 76 include_latest=True)
79 77 expected = [(1, 3, (hist[-1], "spam"))] \
80 78 + [(s, n, (c, None)) for (s, n, c) in newhist]
81 79 assert list(gothist) == expected
82 80
83 81 gothist = ip.history_manager.get_tail(2)
84 82 expected = newhist[-3:-1]
85 83 assert list(gothist) == expected
86 84
87 85 # Check get_hist_search
88 86
89 87 gothist = ip.history_manager.search("*test*")
90 88 assert list(gothist) == [(1, 2, hist[1])]
91 89
92 90 gothist = ip.history_manager.search("*=*")
93 91 assert list(gothist) == [
94 92 (1, 1, hist[0]),
95 93 (1, 2, hist[1]),
96 94 (1, 3, hist[2]),
97 95 newhist[0],
98 96 newhist[2],
99 97 newhist[3],
100 98 ]
101 99
102 100 gothist = ip.history_manager.search("*=*", n=4)
103 101 assert list(gothist) == [
104 102 (1, 3, hist[2]),
105 103 newhist[0],
106 104 newhist[2],
107 105 newhist[3],
108 106 ]
109 107
110 108 gothist = ip.history_manager.search("*=*", unique=True)
111 109 assert list(gothist) == [
112 110 (1, 1, hist[0]),
113 111 (1, 2, hist[1]),
114 112 (1, 3, hist[2]),
115 113 newhist[2],
116 114 newhist[3],
117 115 ]
118 116
119 117 gothist = ip.history_manager.search("*=*", unique=True, n=3)
120 118 assert list(gothist) == [(1, 3, hist[2]), newhist[2], newhist[3]]
121 119
122 120 gothist = ip.history_manager.search("b*", output=True)
123 121 assert list(gothist) == [(1, 3, (hist[2], "spam"))]
124 122
125 123 # Cross testing: check that magic %save can get previous session.
126 124 testfilename = (tmp_path / "test.py").resolve()
127 125 ip.magic("save " + str(testfilename) + " ~1/1-3")
128 126 with io.open(testfilename, encoding="utf-8") as testfile:
129 127 assert testfile.read() == "# coding: utf-8\n" + "\n".join(hist) + "\n"
130 128
131 129 # Duplicate line numbers - check that it doesn't crash, and
132 130 # gets a new session
133 131 ip.history_manager.store_inputs(1, "rogue")
134 132 ip.history_manager.writeout_cache()
135 133 assert ip.history_manager.session_number == 3
134
135 # Check that session and line values are not just max values
136 sessid, lineno, entry = newhist[-1]
137 assert lineno > 1
138 ip.history_manager.reset()
139 lineno = 1
140 ip.history_manager.store_inputs(lineno, entry)
141 gothist = ip.history_manager.search("*=*", unique=True)
142 hist = list(gothist)[-1]
143 assert sessid < hist[0]
144 assert hist[1:] == (lineno, entry)
136 145 finally:
137 146 # Ensure saving thread is shut down before we try to clean up the files
138 147 ip.history_manager.save_thread.stop()
139 148 # Forcibly close database rather than relying on garbage collection
140 149 ip.history_manager.db.close()
141 150 # Restore history manager
142 151 ip.history_manager = hist_manager_ori
143 152
144 153
145 154 def test_extract_hist_ranges():
146 155 instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5 ~10/"
147 156 expected = [(0, 1, 2), # 0 == current session
148 157 (2, 3, 4),
149 158 (-4, 5, 7),
150 159 (-4, 7, 10),
151 160 (-9, 2, None), # None == to end
152 161 (-8, 1, None),
153 162 (-7, 1, 6),
154 163 (-10, 1, None)]
155 164 actual = list(extract_hist_ranges(instr))
156 165 assert actual == expected
157 166
158 167
159 168 def test_extract_hist_ranges_empty_str():
160 169 instr = ""
161 170 expected = [(0, 1, None)] # 0 == current session, None == to end
162 171 actual = list(extract_hist_ranges(instr))
163 172 assert actual == expected
164 173
165 174
166 175 def test_magic_rerun():
167 176 """Simple test for %rerun (no args -> rerun last line)"""
168 177 ip = get_ipython()
169 178 ip.run_cell("a = 10", store_history=True)
170 179 ip.run_cell("a += 1", store_history=True)
171 180 assert ip.user_ns["a"] == 11
172 181 ip.run_cell("%rerun", store_history=True)
173 182 assert ip.user_ns["a"] == 12
174 183
175 184 def test_timestamp_type():
176 185 ip = get_ipython()
177 186 info = ip.history_manager.get_session_info()
178 187 assert isinstance(info[1], datetime)
179 188
180 189 def test_hist_file_config():
181 190 cfg = Config()
182 191 tfile = tempfile.NamedTemporaryFile(delete=False)
183 192 cfg.HistoryManager.hist_file = Path(tfile.name)
184 193 try:
185 194 hm = HistoryManager(shell=get_ipython(), config=cfg)
186 195 assert hm.hist_file == cfg.HistoryManager.hist_file
187 196 finally:
188 197 try:
189 198 Path(tfile.name).unlink()
190 199 except OSError:
191 200 # same catch as in testing.tools.TempFileMixin
192 201 # On Windows, even though we close the file, we still can't
193 202 # delete it. I have no clue why
194 203 pass
195 204
196 205 def test_histmanager_disabled():
197 206 """Ensure that disabling the history manager doesn't create a database."""
198 207 cfg = Config()
199 208 cfg.HistoryAccessor.enabled = False
200 209
201 210 ip = get_ipython()
202 211 with TemporaryDirectory() as tmpdir:
203 212 hist_manager_ori = ip.history_manager
204 213 hist_file = Path(tmpdir) / "history.sqlite"
205 214 cfg.HistoryManager.hist_file = hist_file
206 215 try:
207 216 ip.history_manager = HistoryManager(shell=ip, config=cfg)
208 217 hist = ["a=1", "def f():\n test = 1\n return test", "b='€Æ¾÷ß'"]
209 218 for i, h in enumerate(hist, start=1):
210 219 ip.history_manager.store_inputs(i, h)
211 220 assert ip.history_manager.input_hist_raw == [""] + hist
212 221 ip.history_manager.reset()
213 222 ip.history_manager.end_session()
214 223 finally:
215 224 ip.history_manager = hist_manager_ori
216 225
217 226 # hist_file should not be created
218 227 assert hist_file.exists() is False
General Comments 0
You need to be logged in to leave comments. Login now