##// END OF EJS Templates
Fix DeprecationWarning on 3.12 with a datetime adapter for sqlite3...
Sergey B Kirpichev -
Show More
@@ -1,972 +1,981 b''
1 1 """ History related magics and functionality """
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6
7 7 import atexit
8 8 import datetime
9 9 from pathlib import Path
10 10 import re
11 11 import sqlite3
12 import sys
12 13 import threading
13 14
14 15 from traitlets.config.configurable import LoggingConfigurable
15 16 from decorator import decorator
16 17 from IPython.utils.decorators import undoc
17 18 from IPython.paths import locate_profile
18 19 from traitlets import (
19 20 Any,
20 21 Bool,
21 22 Dict,
22 23 Instance,
23 24 Integer,
24 25 List,
25 26 Unicode,
26 27 Union,
27 28 TraitError,
28 29 default,
29 30 observe,
30 31 )
31 32
33
34 if sys.version_info >= (3, 12):
35
36 def _adapt_datetime(val):
37 return val.isoformat(" ")
38
39 sqlite3.register_adapter(datetime.datetime, _adapt_datetime)
40
32 41 #-----------------------------------------------------------------------------
33 42 # Classes and functions
34 43 #-----------------------------------------------------------------------------
35 44
36 45 @undoc
37 46 class DummyDB(object):
38 47 """Dummy DB that will act as a black hole for history.
39 48
40 49 Only used in the absence of sqlite"""
41 50 def execute(*args, **kwargs):
42 51 return []
43 52
44 53 def commit(self, *args, **kwargs):
45 54 pass
46 55
47 56 def __enter__(self, *args, **kwargs):
48 57 pass
49 58
50 59 def __exit__(self, *args, **kwargs):
51 60 pass
52 61
53 62
54 63 @decorator
55 64 def only_when_enabled(f, self, *a, **kw):
56 65 """Decorator: return an empty list in the absence of sqlite."""
57 66 if not self.enabled:
58 67 return []
59 68 else:
60 69 return f(self, *a, **kw)
61 70
62 71
63 72 # use 16kB as threshold for whether a corrupt history db should be saved
64 73 # that should be at least 100 entries or so
65 74 _SAVE_DB_SIZE = 16384
66 75
67 76 @decorator
68 77 def catch_corrupt_db(f, self, *a, **kw):
69 78 """A decorator which wraps HistoryAccessor method calls to catch errors from
70 79 a corrupt SQLite database, move the old database out of the way, and create
71 80 a new one.
72 81
73 82 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
74 83 not just a corrupt file.
75 84 """
76 85 try:
77 86 return f(self, *a, **kw)
78 87 except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
79 88 self._corrupt_db_counter += 1
80 89 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
81 90 if self.hist_file != ':memory:':
82 91 if self._corrupt_db_counter > self._corrupt_db_limit:
83 92 self.hist_file = ':memory:'
84 93 self.log.error("Failed to load history too many times, history will not be saved.")
85 94 elif self.hist_file.is_file():
86 95 # move the file out of the way
87 96 base = str(self.hist_file.parent / self.hist_file.stem)
88 97 ext = self.hist_file.suffix
89 98 size = self.hist_file.stat().st_size
90 99 if size >= _SAVE_DB_SIZE:
91 100 # if there's significant content, avoid clobbering
92 101 now = datetime.datetime.now().isoformat().replace(':', '.')
93 102 newpath = base + '-corrupt-' + now + ext
94 103 # don't clobber previous corrupt backups
95 104 for i in range(100):
96 105 if not Path(newpath).exists():
97 106 break
98 107 else:
99 108 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
100 109 else:
101 110 # not much content, possibly empty; don't worry about clobbering
102 111 # maybe we should just delete it?
103 112 newpath = base + '-corrupt' + ext
104 113 self.hist_file.rename(newpath)
105 114 self.log.error("History file was moved to %s and a new file created.", newpath)
106 115 self.init_db()
107 116 return []
108 117 else:
109 118 # Failed with :memory:, something serious is wrong
110 119 raise
111 120
112 121
113 122 class HistoryAccessorBase(LoggingConfigurable):
114 123 """An abstract class for History Accessors """
115 124
116 125 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
117 126 raise NotImplementedError
118 127
119 128 def search(self, pattern="*", raw=True, search_raw=True,
120 129 output=False, n=None, unique=False):
121 130 raise NotImplementedError
122 131
123 132 def get_range(self, session, start=1, stop=None, raw=True,output=False):
124 133 raise NotImplementedError
125 134
126 135 def get_range_by_str(self, rangestr, raw=True, output=False):
127 136 raise NotImplementedError
128 137
129 138
130 139 class HistoryAccessor(HistoryAccessorBase):
131 140 """Access the history database without adding to it.
132 141
133 142 This is intended for use by standalone history tools. IPython shells use
134 143 HistoryManager, below, which is a subclass of this."""
135 144
136 145 # counter for init_db retries, so we don't keep trying over and over
137 146 _corrupt_db_counter = 0
138 147 # after two failures, fallback on :memory:
139 148 _corrupt_db_limit = 2
140 149
141 150 # String holding the path to the history file
142 151 hist_file = Union(
143 152 [Instance(Path), Unicode()],
144 153 help="""Path to file to use for SQLite history database.
145 154
146 155 By default, IPython will put the history database in the IPython
147 156 profile directory. If you would rather share one history among
148 157 profiles, you can set this value in each, so that they are consistent.
149 158
150 159 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
151 160 mounts. If you see IPython hanging, try setting this to something on a
152 161 local disk, e.g::
153 162
154 163 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
155 164
156 165 you can also use the specific value `:memory:` (including the colon
157 166 at both end but not the back ticks), to avoid creating an history file.
158 167
159 168 """,
160 169 ).tag(config=True)
161 170
162 171 enabled = Bool(True,
163 172 help="""enable the SQLite history
164 173
165 174 set enabled=False to disable the SQLite history,
166 175 in which case there will be no stored history, no SQLite connection,
167 176 and no background saving thread. This may be necessary in some
168 177 threaded environments where IPython is embedded.
169 178 """,
170 179 ).tag(config=True)
171 180
172 181 connection_options = Dict(
173 182 help="""Options for configuring the SQLite connection
174 183
175 184 These options are passed as keyword args to sqlite3.connect
176 185 when establishing database connections.
177 186 """
178 187 ).tag(config=True)
179 188
180 189 @default("connection_options")
181 190 def _default_connection_options(self):
182 191 return dict(check_same_thread=False)
183 192
184 193 # The SQLite database
185 194 db = Any()
186 195 @observe('db')
187 196 def _db_changed(self, change):
188 197 """validate the db, since it can be an Instance of two different types"""
189 198 new = change['new']
190 199 connection_types = (DummyDB, sqlite3.Connection)
191 200 if not isinstance(new, connection_types):
192 201 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
193 202 (self.__class__.__name__, new)
194 203 raise TraitError(msg)
195 204
196 205 def __init__(self, profile="default", hist_file="", **traits):
197 206 """Create a new history accessor.
198 207
199 208 Parameters
200 209 ----------
201 210 profile : str
202 211 The name of the profile from which to open history.
203 212 hist_file : str
204 213 Path to an SQLite history database stored by IPython. If specified,
205 214 hist_file overrides profile.
206 215 config : :class:`~traitlets.config.loader.Config`
207 216 Config object. hist_file can also be set through this.
208 217 """
209 218 super(HistoryAccessor, self).__init__(**traits)
210 219 # defer setting hist_file from kwarg until after init,
211 220 # otherwise the default kwarg value would clobber any value
212 221 # set by config
213 222 if hist_file:
214 223 self.hist_file = hist_file
215 224
216 225 try:
217 226 self.hist_file
218 227 except TraitError:
219 228 # No one has set the hist_file, yet.
220 229 self.hist_file = self._get_hist_file_name(profile)
221 230
222 231 self.init_db()
223 232
224 233 def _get_hist_file_name(self, profile='default'):
225 234 """Find the history file for the given profile name.
226 235
227 236 This is overridden by the HistoryManager subclass, to use the shell's
228 237 active profile.
229 238
230 239 Parameters
231 240 ----------
232 241 profile : str
233 242 The name of a profile which has a history file.
234 243 """
235 244 return Path(locate_profile(profile)) / "history.sqlite"
236 245
237 246 @catch_corrupt_db
238 247 def init_db(self):
239 248 """Connect to the database, and create tables if necessary."""
240 249 if not self.enabled:
241 250 self.db = DummyDB()
242 251 return
243 252
244 253 # use detect_types so that timestamps return datetime objects
245 254 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
246 255 kwargs.update(self.connection_options)
247 256 self.db = sqlite3.connect(str(self.hist_file), **kwargs)
248 257 with self.db:
249 258 self.db.execute(
250 259 """CREATE TABLE IF NOT EXISTS sessions (session integer
251 260 primary key autoincrement, start timestamp,
252 261 end timestamp, num_cmds integer, remark text)"""
253 262 )
254 263 self.db.execute(
255 264 """CREATE TABLE IF NOT EXISTS history
256 265 (session integer, line integer, source text, source_raw text,
257 266 PRIMARY KEY (session, line))"""
258 267 )
259 268 # Output history is optional, but ensure the table's there so it can be
260 269 # enabled later.
261 270 self.db.execute(
262 271 """CREATE TABLE IF NOT EXISTS output_history
263 272 (session integer, line integer, output text,
264 273 PRIMARY KEY (session, line))"""
265 274 )
266 275 # success! reset corrupt db count
267 276 self._corrupt_db_counter = 0
268 277
269 278 def writeout_cache(self):
270 279 """Overridden by HistoryManager to dump the cache before certain
271 280 database lookups."""
272 281 pass
273 282
274 283 ## -------------------------------
275 284 ## Methods for retrieving history:
276 285 ## -------------------------------
277 286 def _run_sql(self, sql, params, raw=True, output=False, latest=False):
278 287 """Prepares and runs an SQL query for the history database.
279 288
280 289 Parameters
281 290 ----------
282 291 sql : str
283 292 Any filtering expressions to go after SELECT ... FROM ...
284 293 params : tuple
285 294 Parameters passed to the SQL query (to replace "?")
286 295 raw, output : bool
287 296 See :meth:`get_range`
288 297 latest : bool
289 298 Select rows with max (session, line)
290 299
291 300 Returns
292 301 -------
293 302 Tuples as :meth:`get_range`
294 303 """
295 304 toget = 'source_raw' if raw else 'source'
296 305 sqlfrom = "history"
297 306 if output:
298 307 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
299 308 toget = "history.%s, output_history.output" % toget
300 309 if latest:
301 310 toget += ", MAX(session * 128 * 1024 + line)"
302 311 this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql
303 312 cur = self.db.execute(this_querry, params)
304 313 if latest:
305 314 cur = (row[:-1] for row in cur)
306 315 if output: # Regroup into 3-tuples, and parse JSON
307 316 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
308 317 return cur
309 318
310 319 @only_when_enabled
311 320 @catch_corrupt_db
312 321 def get_session_info(self, session):
313 322 """Get info about a session.
314 323
315 324 Parameters
316 325 ----------
317 326 session : int
318 327 Session number to retrieve.
319 328
320 329 Returns
321 330 -------
322 331 session_id : int
323 332 Session ID number
324 333 start : datetime
325 334 Timestamp for the start of the session.
326 335 end : datetime
327 336 Timestamp for the end of the session, or None if IPython crashed.
328 337 num_cmds : int
329 338 Number of commands run, or None if IPython crashed.
330 339 remark : unicode
331 340 A manually set description.
332 341 """
333 342 query = "SELECT * from sessions where session == ?"
334 343 return self.db.execute(query, (session,)).fetchone()
335 344
336 345 @catch_corrupt_db
337 346 def get_last_session_id(self):
338 347 """Get the last session ID currently in the database.
339 348
340 349 Within IPython, this should be the same as the value stored in
341 350 :attr:`HistoryManager.session_number`.
342 351 """
343 352 for record in self.get_tail(n=1, include_latest=True):
344 353 return record[0]
345 354
346 355 @catch_corrupt_db
347 356 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
348 357 """Get the last n lines from the history database.
349 358
350 359 Parameters
351 360 ----------
352 361 n : int
353 362 The number of lines to get
354 363 raw, output : bool
355 364 See :meth:`get_range`
356 365 include_latest : bool
357 366 If False (default), n+1 lines are fetched, and the latest one
358 367 is discarded. This is intended to be used where the function
359 368 is called by a user command, which it should not return.
360 369
361 370 Returns
362 371 -------
363 372 Tuples as :meth:`get_range`
364 373 """
365 374 self.writeout_cache()
366 375 if not include_latest:
367 376 n += 1
368 377 cur = self._run_sql(
369 378 "ORDER BY session DESC, line DESC LIMIT ?", (n,), raw=raw, output=output
370 379 )
371 380 if not include_latest:
372 381 return reversed(list(cur)[1:])
373 382 return reversed(list(cur))
374 383
375 384 @catch_corrupt_db
376 385 def search(self, pattern="*", raw=True, search_raw=True,
377 386 output=False, n=None, unique=False):
378 387 """Search the database using unix glob-style matching (wildcards
379 388 * and ?).
380 389
381 390 Parameters
382 391 ----------
383 392 pattern : str
384 393 The wildcarded pattern to match when searching
385 394 search_raw : bool
386 395 If True, search the raw input, otherwise, the parsed input
387 396 raw, output : bool
388 397 See :meth:`get_range`
389 398 n : None or int
390 399 If an integer is given, it defines the limit of
391 400 returned entries.
392 401 unique : bool
393 402 When it is true, return only unique entries.
394 403
395 404 Returns
396 405 -------
397 406 Tuples as :meth:`get_range`
398 407 """
399 408 tosearch = "source_raw" if search_raw else "source"
400 409 if output:
401 410 tosearch = "history." + tosearch
402 411 self.writeout_cache()
403 412 sqlform = "WHERE %s GLOB ?" % tosearch
404 413 params = (pattern,)
405 414 if unique:
406 415 sqlform += ' GROUP BY {0}'.format(tosearch)
407 416 if n is not None:
408 417 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
409 418 params += (n,)
410 419 elif unique:
411 420 sqlform += " ORDER BY session, line"
412 421 cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
413 422 if n is not None:
414 423 return reversed(list(cur))
415 424 return cur
416 425
417 426 @catch_corrupt_db
418 427 def get_range(self, session, start=1, stop=None, raw=True,output=False):
419 428 """Retrieve input by session.
420 429
421 430 Parameters
422 431 ----------
423 432 session : int
424 433 Session number to retrieve.
425 434 start : int
426 435 First line to retrieve.
427 436 stop : int
428 437 End of line range (excluded from output itself). If None, retrieve
429 438 to the end of the session.
430 439 raw : bool
431 440 If True, return untranslated input
432 441 output : bool
433 442 If True, attempt to include output. This will be 'real' Python
434 443 objects for the current session, or text reprs from previous
435 444 sessions if db_log_output was enabled at the time. Where no output
436 445 is found, None is used.
437 446
438 447 Returns
439 448 -------
440 449 entries
441 450 An iterator over the desired lines. Each line is a 3-tuple, either
442 451 (session, line, input) if output is False, or
443 452 (session, line, (input, output)) if output is True.
444 453 """
445 454 if stop:
446 455 lineclause = "line >= ? AND line < ?"
447 456 params = (session, start, stop)
448 457 else:
449 458 lineclause = "line>=?"
450 459 params = (session, start)
451 460
452 461 return self._run_sql("WHERE session==? AND %s" % lineclause,
453 462 params, raw=raw, output=output)
454 463
455 464 def get_range_by_str(self, rangestr, raw=True, output=False):
456 465 """Get lines of history from a string of ranges, as used by magic
457 466 commands %hist, %save, %macro, etc.
458 467
459 468 Parameters
460 469 ----------
461 470 rangestr : str
462 471 A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
463 472 this will return everything from current session's history.
464 473
465 474 See the documentation of :func:`%history` for the full details.
466 475
467 476 raw, output : bool
468 477 As :meth:`get_range`
469 478
470 479 Returns
471 480 -------
472 481 Tuples as :meth:`get_range`
473 482 """
474 483 for sess, s, e in extract_hist_ranges(rangestr):
475 484 for line in self.get_range(sess, s, e, raw=raw, output=output):
476 485 yield line
477 486
478 487
479 488 class HistoryManager(HistoryAccessor):
480 489 """A class to organize all history-related functionality in one place.
481 490 """
482 491 # Public interface
483 492
484 493 # An instance of the IPython shell we are attached to
485 494 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
486 495 allow_none=True)
487 496 # Lists to hold processed and raw history. These start with a blank entry
488 497 # so that we can index them starting from 1
489 498 input_hist_parsed = List([""])
490 499 input_hist_raw = List([""])
491 500 # A list of directories visited during session
492 501 dir_hist = List()
493 502 @default('dir_hist')
494 503 def _dir_hist_default(self):
495 504 try:
496 505 return [Path.cwd()]
497 506 except OSError:
498 507 return []
499 508
500 509 # A dict of output history, keyed with ints from the shell's
501 510 # execution count.
502 511 output_hist = Dict()
503 512 # The text/plain repr of outputs.
504 513 output_hist_reprs = Dict()
505 514
506 515 # The number of the current session in the history database
507 516 session_number = Integer()
508 517
509 518 db_log_output = Bool(False,
510 519 help="Should the history database include output? (default: no)"
511 520 ).tag(config=True)
512 521 db_cache_size = Integer(0,
513 522 help="Write to database every x commands (higher values save disk access & power).\n"
514 523 "Values of 1 or less effectively disable caching."
515 524 ).tag(config=True)
516 525 # The input and output caches
517 526 db_input_cache = List()
518 527 db_output_cache = List()
519 528
520 529 # History saving in separate thread
521 530 save_thread = Instance('IPython.core.history.HistorySavingThread',
522 531 allow_none=True)
523 532 save_flag = Instance(threading.Event, allow_none=True)
524 533
525 534 # Private interface
526 535 # Variables used to store the three last inputs from the user. On each new
527 536 # history update, we populate the user's namespace with these, shifted as
528 537 # necessary.
529 538 _i00 = Unicode(u'')
530 539 _i = Unicode(u'')
531 540 _ii = Unicode(u'')
532 541 _iii = Unicode(u'')
533 542
534 543 # A regex matching all forms of the exit command, so that we don't store
535 544 # them in the history (it's annoying to rewind the first entry and land on
536 545 # an exit call).
537 546 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
538 547
539 548 def __init__(self, shell=None, config=None, **traits):
540 549 """Create a new history manager associated with a shell instance.
541 550 """
542 551 super(HistoryManager, self).__init__(shell=shell, config=config,
543 552 **traits)
544 553 self.save_flag = threading.Event()
545 554 self.db_input_cache_lock = threading.Lock()
546 555 self.db_output_cache_lock = threading.Lock()
547 556
548 557 try:
549 558 self.new_session()
550 559 except sqlite3.OperationalError:
551 560 self.log.error("Failed to create history session in %s. History will not be saved.",
552 561 self.hist_file, exc_info=True)
553 562 self.hist_file = ':memory:'
554 563
555 564 if self.enabled and self.hist_file != ':memory:':
556 565 self.save_thread = HistorySavingThread(self)
557 566 self.save_thread.start()
558 567
559 568 def _get_hist_file_name(self, profile=None):
560 569 """Get default history file name based on the Shell's profile.
561 570
562 571 The profile parameter is ignored, but must exist for compatibility with
563 572 the parent class."""
564 573 profile_dir = self.shell.profile_dir.location
565 574 return Path(profile_dir) / "history.sqlite"
566 575
567 576 @only_when_enabled
568 577 def new_session(self, conn=None):
569 578 """Get a new session number."""
570 579 if conn is None:
571 580 conn = self.db
572 581
573 582 with conn:
574 583 cur = conn.execute(
575 584 """INSERT INTO sessions VALUES (NULL, ?, NULL,
576 585 NULL, '') """,
577 586 (datetime.datetime.now(),),
578 587 )
579 588 self.session_number = cur.lastrowid
580 589
581 590 def end_session(self):
582 591 """Close the database session, filling in the end time and line count."""
583 592 self.writeout_cache()
584 593 with self.db:
585 594 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
586 595 session==?""", (datetime.datetime.now(),
587 596 len(self.input_hist_parsed)-1, self.session_number))
588 597 self.session_number = 0
589 598
590 599 def name_session(self, name):
591 600 """Give the current session a name in the history database."""
592 601 with self.db:
593 602 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
594 603 (name, self.session_number))
595 604
596 605 def reset(self, new_session=True):
597 606 """Clear the session history, releasing all object references, and
598 607 optionally open a new session."""
599 608 self.output_hist.clear()
600 609 # The directory history can't be completely empty
601 610 self.dir_hist[:] = [Path.cwd()]
602 611
603 612 if new_session:
604 613 if self.session_number:
605 614 self.end_session()
606 615 self.input_hist_parsed[:] = [""]
607 616 self.input_hist_raw[:] = [""]
608 617 self.new_session()
609 618
610 619 # ------------------------------
611 620 # Methods for retrieving history
612 621 # ------------------------------
613 622 def get_session_info(self, session=0):
614 623 """Get info about a session.
615 624
616 625 Parameters
617 626 ----------
618 627 session : int
619 628 Session number to retrieve. The current session is 0, and negative
620 629 numbers count back from current session, so -1 is the previous session.
621 630
622 631 Returns
623 632 -------
624 633 session_id : int
625 634 Session ID number
626 635 start : datetime
627 636 Timestamp for the start of the session.
628 637 end : datetime
629 638 Timestamp for the end of the session, or None if IPython crashed.
630 639 num_cmds : int
631 640 Number of commands run, or None if IPython crashed.
632 641 remark : unicode
633 642 A manually set description.
634 643 """
635 644 if session <= 0:
636 645 session += self.session_number
637 646
638 647 return super(HistoryManager, self).get_session_info(session=session)
639 648
640 649 @catch_corrupt_db
641 650 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
642 651 """Get the last n lines from the history database.
643 652
644 653 Most recent entry last.
645 654
646 655 Completion will be reordered so that that the last ones are when
647 656 possible from current session.
648 657
649 658 Parameters
650 659 ----------
651 660 n : int
652 661 The number of lines to get
653 662 raw, output : bool
654 663 See :meth:`get_range`
655 664 include_latest : bool
656 665 If False (default), n+1 lines are fetched, and the latest one
657 666 is discarded. This is intended to be used where the function
658 667 is called by a user command, which it should not return.
659 668
660 669 Returns
661 670 -------
662 671 Tuples as :meth:`get_range`
663 672 """
664 673 self.writeout_cache()
665 674 if not include_latest:
666 675 n += 1
667 676 # cursor/line/entry
668 677 this_cur = list(
669 678 self._run_sql(
670 679 "WHERE session == ? ORDER BY line DESC LIMIT ? ",
671 680 (self.session_number, n),
672 681 raw=raw,
673 682 output=output,
674 683 )
675 684 )
676 685 other_cur = list(
677 686 self._run_sql(
678 687 "WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?",
679 688 (self.session_number, n),
680 689 raw=raw,
681 690 output=output,
682 691 )
683 692 )
684 693
685 694 everything = this_cur + other_cur
686 695
687 696 everything = everything[:n]
688 697
689 698 if not include_latest:
690 699 return list(everything)[:0:-1]
691 700 return list(everything)[::-1]
692 701
693 702 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
694 703 """Get input and output history from the current session. Called by
695 704 get_range, and takes similar parameters."""
696 705 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
697 706
698 707 n = len(input_hist)
699 708 if start < 0:
700 709 start += n
701 710 if not stop or (stop > n):
702 711 stop = n
703 712 elif stop < 0:
704 713 stop += n
705 714
706 715 for i in range(start, stop):
707 716 if output:
708 717 line = (input_hist[i], self.output_hist_reprs.get(i))
709 718 else:
710 719 line = input_hist[i]
711 720 yield (0, i, line)
712 721
713 722 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
714 723 """Retrieve input by session.
715 724
716 725 Parameters
717 726 ----------
718 727 session : int
719 728 Session number to retrieve. The current session is 0, and negative
720 729 numbers count back from current session, so -1 is previous session.
721 730 start : int
722 731 First line to retrieve.
723 732 stop : int
724 733 End of line range (excluded from output itself). If None, retrieve
725 734 to the end of the session.
726 735 raw : bool
727 736 If True, return untranslated input
728 737 output : bool
729 738 If True, attempt to include output. This will be 'real' Python
730 739 objects for the current session, or text reprs from previous
731 740 sessions if db_log_output was enabled at the time. Where no output
732 741 is found, None is used.
733 742
734 743 Returns
735 744 -------
736 745 entries
737 746 An iterator over the desired lines. Each line is a 3-tuple, either
738 747 (session, line, input) if output is False, or
739 748 (session, line, (input, output)) if output is True.
740 749 """
741 750 if session <= 0:
742 751 session += self.session_number
743 752 if session==self.session_number: # Current session
744 753 return self._get_range_session(start, stop, raw, output)
745 754 return super(HistoryManager, self).get_range(session, start, stop, raw,
746 755 output)
747 756
748 757 ## ----------------------------
749 758 ## Methods for storing history:
750 759 ## ----------------------------
751 760 def store_inputs(self, line_num, source, source_raw=None):
752 761 """Store source and raw input in history and create input cache
753 762 variables ``_i*``.
754 763
755 764 Parameters
756 765 ----------
757 766 line_num : int
758 767 The prompt number of this input.
759 768 source : str
760 769 Python input.
761 770 source_raw : str, optional
762 771 If given, this is the raw input without any IPython transformations
763 772 applied to it. If not given, ``source`` is used.
764 773 """
765 774 if source_raw is None:
766 775 source_raw = source
767 776 source = source.rstrip('\n')
768 777 source_raw = source_raw.rstrip('\n')
769 778
770 779 # do not store exit/quit commands
771 780 if self._exit_re.match(source_raw.strip()):
772 781 return
773 782
774 783 self.input_hist_parsed.append(source)
775 784 self.input_hist_raw.append(source_raw)
776 785
777 786 with self.db_input_cache_lock:
778 787 self.db_input_cache.append((line_num, source, source_raw))
779 788 # Trigger to flush cache and write to DB.
780 789 if len(self.db_input_cache) >= self.db_cache_size:
781 790 self.save_flag.set()
782 791
783 792 # update the auto _i variables
784 793 self._iii = self._ii
785 794 self._ii = self._i
786 795 self._i = self._i00
787 796 self._i00 = source_raw
788 797
789 798 # hackish access to user namespace to create _i1,_i2... dynamically
790 799 new_i = '_i%s' % line_num
791 800 to_main = {'_i': self._i,
792 801 '_ii': self._ii,
793 802 '_iii': self._iii,
794 803 new_i : self._i00 }
795 804
796 805 if self.shell is not None:
797 806 self.shell.push(to_main, interactive=False)
798 807
799 808 def store_output(self, line_num):
800 809 """If database output logging is enabled, this saves all the
801 810 outputs from the indicated prompt number to the database. It's
802 811 called by run_cell after code has been executed.
803 812
804 813 Parameters
805 814 ----------
806 815 line_num : int
807 816 The line number from which to save outputs
808 817 """
809 818 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
810 819 return
811 820 output = self.output_hist_reprs[line_num]
812 821
813 822 with self.db_output_cache_lock:
814 823 self.db_output_cache.append((line_num, output))
815 824 if self.db_cache_size <= 1:
816 825 self.save_flag.set()
817 826
818 827 def _writeout_input_cache(self, conn):
819 828 with conn:
820 829 for line in self.db_input_cache:
821 830 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
822 831 (self.session_number,)+line)
823 832
824 833 def _writeout_output_cache(self, conn):
825 834 with conn:
826 835 for line in self.db_output_cache:
827 836 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
828 837 (self.session_number,)+line)
829 838
830 839 @only_when_enabled
831 840 def writeout_cache(self, conn=None):
832 841 """Write any entries in the cache to the database."""
833 842 if conn is None:
834 843 conn = self.db
835 844
836 845 with self.db_input_cache_lock:
837 846 try:
838 847 self._writeout_input_cache(conn)
839 848 except sqlite3.IntegrityError:
840 849 self.new_session(conn)
841 850 print("ERROR! Session/line number was not unique in",
842 851 "database. History logging moved to new session",
843 852 self.session_number)
844 853 try:
845 854 # Try writing to the new session. If this fails, don't
846 855 # recurse
847 856 self._writeout_input_cache(conn)
848 857 except sqlite3.IntegrityError:
849 858 pass
850 859 finally:
851 860 self.db_input_cache = []
852 861
853 862 with self.db_output_cache_lock:
854 863 try:
855 864 self._writeout_output_cache(conn)
856 865 except sqlite3.IntegrityError:
857 866 print("!! Session/line number for output was not unique",
858 867 "in database. Output will not be stored.")
859 868 finally:
860 869 self.db_output_cache = []
861 870
862 871
863 872 class HistorySavingThread(threading.Thread):
864 873 """This thread takes care of writing history to the database, so that
865 874 the UI isn't held up while that happens.
866 875
867 876 It waits for the HistoryManager's save_flag to be set, then writes out
868 877 the history cache. The main thread is responsible for setting the flag when
869 878 the cache size reaches a defined threshold."""
870 879 daemon = True
871 880 stop_now = False
872 881 enabled = True
873 882 def __init__(self, history_manager):
874 883 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
875 884 self.history_manager = history_manager
876 885 self.enabled = history_manager.enabled
877 886 atexit.register(self.stop)
878 887
879 888 @only_when_enabled
880 889 def run(self):
881 890 # We need a separate db connection per thread:
882 891 try:
883 892 self.db = sqlite3.connect(
884 893 str(self.history_manager.hist_file),
885 894 **self.history_manager.connection_options,
886 895 )
887 896 while True:
888 897 self.history_manager.save_flag.wait()
889 898 if self.stop_now:
890 899 self.db.close()
891 900 return
892 901 self.history_manager.save_flag.clear()
893 902 self.history_manager.writeout_cache(self.db)
894 903 except Exception as e:
895 904 print(("The history saving thread hit an unexpected error (%s)."
896 905 "History will not be written to the database.") % repr(e))
897 906
898 907 def stop(self):
899 908 """This can be called from the main thread to safely stop this thread.
900 909
901 910 Note that it does not attempt to write out remaining history before
902 911 exiting. That should be done by calling the HistoryManager's
903 912 end_session method."""
904 913 self.stop_now = True
905 914 self.history_manager.save_flag.set()
906 915 self.join()
907 916
908 917
909 918 # To match, e.g. ~5/8-~2/3
910 919 range_re = re.compile(r"""
911 920 ((?P<startsess>~?\d+)/)?
912 921 (?P<start>\d+)?
913 922 ((?P<sep>[\-:])
914 923 ((?P<endsess>~?\d+)/)?
915 924 (?P<end>\d+))?
916 925 $""", re.VERBOSE)
917 926
918 927
919 928 def extract_hist_ranges(ranges_str):
920 929 """Turn a string of history ranges into 3-tuples of (session, start, stop).
921 930
922 931 Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
923 932 session".
924 933
925 934 Examples
926 935 --------
927 936 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
928 937 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
929 938 """
930 939 if ranges_str == "":
931 940 yield (0, 1, None) # Everything from current session
932 941 return
933 942
934 943 for range_str in ranges_str.split():
935 944 rmatch = range_re.match(range_str)
936 945 if not rmatch:
937 946 continue
938 947 start = rmatch.group("start")
939 948 if start:
940 949 start = int(start)
941 950 end = rmatch.group("end")
942 951 # If no end specified, get (a, a + 1)
943 952 end = int(end) if end else start + 1
944 953 else: # start not specified
945 954 if not rmatch.group('startsess'): # no startsess
946 955 continue
947 956 start = 1
948 957 end = None # provide the entire session hist
949 958
950 959 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
951 960 end += 1
952 961 startsess = rmatch.group("startsess") or "0"
953 962 endsess = rmatch.group("endsess") or startsess
954 963 startsess = int(startsess.replace("~","-"))
955 964 endsess = int(endsess.replace("~","-"))
956 965 assert endsess >= startsess, "start session must be earlier than end session"
957 966
958 967 if endsess == startsess:
959 968 yield (startsess, start, end)
960 969 continue
961 970 # Multiple sessions in one range:
962 971 yield (startsess, start, None)
963 972 for sess in range(startsess+1, endsess):
964 973 yield (sess, 1, None)
965 974 yield (endsess, 1, end)
966 975
967 976
968 977 def _format_lineno(session, line):
969 978 """Helper function to format line numbers properly."""
970 979 if session == 0:
971 980 return str(line)
972 981 return "%s#%s" % (session, line)
General Comments 0
You need to be logged in to leave comments. Login now