##// END OF EJS Templates
Merge pull request #9687 from Carreau/memory-history...
Thomas Kluyver -
r22637:ce175464 merge
parent child Browse files
Show More
@@ -1,908 +1,911 b''
1 1 """ History related magics and functionality """
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6 from __future__ import print_function
7 7
8 8 import atexit
9 9 import datetime
10 10 import os
11 11 import re
12 12 try:
13 13 import sqlite3
14 14 except ImportError:
15 15 try:
16 16 from pysqlite2 import dbapi2 as sqlite3
17 17 except ImportError:
18 18 sqlite3 = None
19 19 import threading
20 20
21 21 from traitlets.config.configurable import LoggingConfigurable
22 22 from decorator import decorator
23 23 from IPython.utils.decorators import undoc
24 24 from IPython.utils.path import locate_profile
25 25 from IPython.utils import py3compat
26 26 from traitlets import (
27 27 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
28 28 default, observe,
29 29 )
30 30 from warnings import warn
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Classes and functions
34 34 #-----------------------------------------------------------------------------
35 35
36 36 @undoc
37 37 class DummyDB(object):
38 38 """Dummy DB that will act as a black hole for history.
39 39
40 40 Only used in the absence of sqlite"""
41 41 def execute(*args, **kwargs):
42 42 return []
43 43
44 44 def commit(self, *args, **kwargs):
45 45 pass
46 46
47 47 def __enter__(self, *args, **kwargs):
48 48 pass
49 49
50 50 def __exit__(self, *args, **kwargs):
51 51 pass
52 52
53 53
54 54 @decorator
55 55 def needs_sqlite(f, self, *a, **kw):
56 56 """Decorator: return an empty list in the absence of sqlite."""
57 57 if sqlite3 is None or not self.enabled:
58 58 return []
59 59 else:
60 60 return f(self, *a, **kw)
61 61
62 62
63 63 if sqlite3 is not None:
64 64 DatabaseError = sqlite3.DatabaseError
65 65 OperationalError = sqlite3.OperationalError
66 66 else:
67 67 @undoc
68 68 class DatabaseError(Exception):
69 69 "Dummy exception when sqlite could not be imported. Should never occur."
70 70
71 71 @undoc
72 72 class OperationalError(Exception):
73 73 "Dummy exception when sqlite could not be imported. Should never occur."
74 74
75 75 # use 16kB as threshold for whether a corrupt history db should be saved
76 76 # that should be at least 100 entries or so
77 77 _SAVE_DB_SIZE = 16384
78 78
79 79 @decorator
80 80 def catch_corrupt_db(f, self, *a, **kw):
81 81 """A decorator which wraps HistoryAccessor method calls to catch errors from
82 82 a corrupt SQLite database, move the old database out of the way, and create
83 83 a new one.
84 84
85 85 We avoid clobbering larger databases because this may be triggered due to filesystem issues,
86 86 not just a corrupt file.
87 87 """
88 88 try:
89 89 return f(self, *a, **kw)
90 90 except (DatabaseError, OperationalError) as e:
91 91 self._corrupt_db_counter += 1
92 92 self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
93 93 if self.hist_file != ':memory:':
94 94 if self._corrupt_db_counter > self._corrupt_db_limit:
95 95 self.hist_file = ':memory:'
96 96 self.log.error("Failed to load history too many times, history will not be saved.")
97 97 elif os.path.isfile(self.hist_file):
98 98 # move the file out of the way
99 99 base, ext = os.path.splitext(self.hist_file)
100 100 size = os.stat(self.hist_file).st_size
101 101 if size >= _SAVE_DB_SIZE:
102 102 # if there's significant content, avoid clobbering
103 103 now = datetime.datetime.now().isoformat().replace(':', '.')
104 104 newpath = base + '-corrupt-' + now + ext
105 105 # don't clobber previous corrupt backups
106 106 for i in range(100):
107 107 if not os.path.isfile(newpath):
108 108 break
109 109 else:
110 110 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
111 111 else:
112 112 # not much content, possibly empty; don't worry about clobbering
113 113 # maybe we should just delete it?
114 114 newpath = base + '-corrupt' + ext
115 115 os.rename(self.hist_file, newpath)
116 116 self.log.error("History file was moved to %s and a new file created.", newpath)
117 117 self.init_db()
118 118 return []
119 119 else:
120 120 # Failed with :memory:, something serious is wrong
121 121 raise
122 122
123 123 class HistoryAccessorBase(LoggingConfigurable):
124 124 """An abstract class for History Accessors """
125 125
126 126 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
127 127 raise NotImplementedError
128 128
129 129 def search(self, pattern="*", raw=True, search_raw=True,
130 130 output=False, n=None, unique=False):
131 131 raise NotImplementedError
132 132
133 133 def get_range(self, session, start=1, stop=None, raw=True,output=False):
134 134 raise NotImplementedError
135 135
136 136 def get_range_by_str(self, rangestr, raw=True, output=False):
137 137 raise NotImplementedError
138 138
139 139
140 140 class HistoryAccessor(HistoryAccessorBase):
141 141 """Access the history database without adding to it.
142 142
143 143 This is intended for use by standalone history tools. IPython shells use
144 144 HistoryManager, below, which is a subclass of this."""
145 145
146 146 # counter for init_db retries, so we don't keep trying over and over
147 147 _corrupt_db_counter = 0
148 148 # after two failures, fallback on :memory:
149 149 _corrupt_db_limit = 2
150 150
151 151 # String holding the path to the history file
152 152 hist_file = Unicode(
153 153 help="""Path to file to use for SQLite history database.
154 154
155 155 By default, IPython will put the history database in the IPython
156 156 profile directory. If you would rather share one history among
157 157 profiles, you can set this value in each, so that they are consistent.
158 158
159 159 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
160 160 mounts. If you see IPython hanging, try setting this to something on a
161 161 local disk, e.g::
162 162
163 163 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
164 164
165 you can also use the specific value `:memory:` (including the colon
166 at both end but not the back ticks), to avoid creating an history file.
167
165 168 """).tag(config=True)
166 169
167 170 enabled = Bool(True,
168 171 help="""enable the SQLite history
169 172
170 173 set enabled=False to disable the SQLite history,
171 174 in which case there will be no stored history, no SQLite connection,
172 175 and no background saving thread. This may be necessary in some
173 176 threaded environments where IPython is embedded.
174 177 """
175 178 ).tag(config=True)
176 179
177 180 connection_options = Dict(
178 181 help="""Options for configuring the SQLite connection
179 182
180 183 These options are passed as keyword args to sqlite3.connect
181 184 when establishing database conenctions.
182 185 """
183 186 ).tag(config=True)
184 187
185 188 # The SQLite database
186 189 db = Any()
187 190 @observe('db')
188 191 def _db_changed(self, change):
189 192 """validate the db, since it can be an Instance of two different types"""
190 193 new = change['new']
191 194 connection_types = (DummyDB,)
192 195 if sqlite3 is not None:
193 196 connection_types = (DummyDB, sqlite3.Connection)
194 197 if not isinstance(new, connection_types):
195 198 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
196 199 (self.__class__.__name__, new)
197 200 raise TraitError(msg)
198 201
199 202 def __init__(self, profile='default', hist_file=u'', **traits):
200 203 """Create a new history accessor.
201 204
202 205 Parameters
203 206 ----------
204 207 profile : str
205 208 The name of the profile from which to open history.
206 209 hist_file : str
207 210 Path to an SQLite history database stored by IPython. If specified,
208 211 hist_file overrides profile.
209 212 config : :class:`~traitlets.config.loader.Config`
210 213 Config object. hist_file can also be set through this.
211 214 """
212 215 # We need a pointer back to the shell for various tasks.
213 216 super(HistoryAccessor, self).__init__(**traits)
214 217 # defer setting hist_file from kwarg until after init,
215 218 # otherwise the default kwarg value would clobber any value
216 219 # set by config
217 220 if hist_file:
218 221 self.hist_file = hist_file
219 222
220 223 if self.hist_file == u'':
221 224 # No one has set the hist_file, yet.
222 225 self.hist_file = self._get_hist_file_name(profile)
223 226
224 227 if sqlite3 is None and self.enabled:
225 228 warn("IPython History requires SQLite, your history will not be saved")
226 229 self.enabled = False
227 230
228 231 self.init_db()
229 232
230 233 def _get_hist_file_name(self, profile='default'):
231 234 """Find the history file for the given profile name.
232 235
233 236 This is overridden by the HistoryManager subclass, to use the shell's
234 237 active profile.
235 238
236 239 Parameters
237 240 ----------
238 241 profile : str
239 242 The name of a profile which has a history file.
240 243 """
241 244 return os.path.join(locate_profile(profile), 'history.sqlite')
242 245
243 246 @catch_corrupt_db
244 247 def init_db(self):
245 248 """Connect to the database, and create tables if necessary."""
246 249 if not self.enabled:
247 250 self.db = DummyDB()
248 251 return
249 252
250 253 # use detect_types so that timestamps return datetime objects
251 254 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
252 255 kwargs.update(self.connection_options)
253 256 self.db = sqlite3.connect(self.hist_file, **kwargs)
254 257 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
255 258 primary key autoincrement, start timestamp,
256 259 end timestamp, num_cmds integer, remark text)""")
257 260 self.db.execute("""CREATE TABLE IF NOT EXISTS history
258 261 (session integer, line integer, source text, source_raw text,
259 262 PRIMARY KEY (session, line))""")
260 263 # Output history is optional, but ensure the table's there so it can be
261 264 # enabled later.
262 265 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
263 266 (session integer, line integer, output text,
264 267 PRIMARY KEY (session, line))""")
265 268 self.db.commit()
266 269 # success! reset corrupt db count
267 270 self._corrupt_db_counter = 0
268 271
269 272 def writeout_cache(self):
270 273 """Overridden by HistoryManager to dump the cache before certain
271 274 database lookups."""
272 275 pass
273 276
274 277 ## -------------------------------
275 278 ## Methods for retrieving history:
276 279 ## -------------------------------
277 280 def _run_sql(self, sql, params, raw=True, output=False):
278 281 """Prepares and runs an SQL query for the history database.
279 282
280 283 Parameters
281 284 ----------
282 285 sql : str
283 286 Any filtering expressions to go after SELECT ... FROM ...
284 287 params : tuple
285 288 Parameters passed to the SQL query (to replace "?")
286 289 raw, output : bool
287 290 See :meth:`get_range`
288 291
289 292 Returns
290 293 -------
291 294 Tuples as :meth:`get_range`
292 295 """
293 296 toget = 'source_raw' if raw else 'source'
294 297 sqlfrom = "history"
295 298 if output:
296 299 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
297 300 toget = "history.%s, output_history.output" % toget
298 301 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
299 302 (toget, sqlfrom) + sql, params)
300 303 if output: # Regroup into 3-tuples, and parse JSON
301 304 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
302 305 return cur
303 306
304 307 @needs_sqlite
305 308 @catch_corrupt_db
306 309 def get_session_info(self, session):
307 310 """Get info about a session.
308 311
309 312 Parameters
310 313 ----------
311 314
312 315 session : int
313 316 Session number to retrieve.
314 317
315 318 Returns
316 319 -------
317 320
318 321 session_id : int
319 322 Session ID number
320 323 start : datetime
321 324 Timestamp for the start of the session.
322 325 end : datetime
323 326 Timestamp for the end of the session, or None if IPython crashed.
324 327 num_cmds : int
325 328 Number of commands run, or None if IPython crashed.
326 329 remark : unicode
327 330 A manually set description.
328 331 """
329 332 query = "SELECT * from sessions where session == ?"
330 333 return self.db.execute(query, (session,)).fetchone()
331 334
332 335 @catch_corrupt_db
333 336 def get_last_session_id(self):
334 337 """Get the last session ID currently in the database.
335 338
336 339 Within IPython, this should be the same as the value stored in
337 340 :attr:`HistoryManager.session_number`.
338 341 """
339 342 for record in self.get_tail(n=1, include_latest=True):
340 343 return record[0]
341 344
342 345 @catch_corrupt_db
343 346 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
344 347 """Get the last n lines from the history database.
345 348
346 349 Parameters
347 350 ----------
348 351 n : int
349 352 The number of lines to get
350 353 raw, output : bool
351 354 See :meth:`get_range`
352 355 include_latest : bool
353 356 If False (default), n+1 lines are fetched, and the latest one
354 357 is discarded. This is intended to be used where the function
355 358 is called by a user command, which it should not return.
356 359
357 360 Returns
358 361 -------
359 362 Tuples as :meth:`get_range`
360 363 """
361 364 self.writeout_cache()
362 365 if not include_latest:
363 366 n += 1
364 367 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
365 368 (n,), raw=raw, output=output)
366 369 if not include_latest:
367 370 return reversed(list(cur)[1:])
368 371 return reversed(list(cur))
369 372
370 373 @catch_corrupt_db
371 374 def search(self, pattern="*", raw=True, search_raw=True,
372 375 output=False, n=None, unique=False):
373 376 """Search the database using unix glob-style matching (wildcards
374 377 * and ?).
375 378
376 379 Parameters
377 380 ----------
378 381 pattern : str
379 382 The wildcarded pattern to match when searching
380 383 search_raw : bool
381 384 If True, search the raw input, otherwise, the parsed input
382 385 raw, output : bool
383 386 See :meth:`get_range`
384 387 n : None or int
385 388 If an integer is given, it defines the limit of
386 389 returned entries.
387 390 unique : bool
388 391 When it is true, return only unique entries.
389 392
390 393 Returns
391 394 -------
392 395 Tuples as :meth:`get_range`
393 396 """
394 397 tosearch = "source_raw" if search_raw else "source"
395 398 if output:
396 399 tosearch = "history." + tosearch
397 400 self.writeout_cache()
398 401 sqlform = "WHERE %s GLOB ?" % tosearch
399 402 params = (pattern,)
400 403 if unique:
401 404 sqlform += ' GROUP BY {0}'.format(tosearch)
402 405 if n is not None:
403 406 sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
404 407 params += (n,)
405 408 elif unique:
406 409 sqlform += " ORDER BY session, line"
407 410 cur = self._run_sql(sqlform, params, raw=raw, output=output)
408 411 if n is not None:
409 412 return reversed(list(cur))
410 413 return cur
411 414
412 415 @catch_corrupt_db
413 416 def get_range(self, session, start=1, stop=None, raw=True,output=False):
414 417 """Retrieve input by session.
415 418
416 419 Parameters
417 420 ----------
418 421 session : int
419 422 Session number to retrieve.
420 423 start : int
421 424 First line to retrieve.
422 425 stop : int
423 426 End of line range (excluded from output itself). If None, retrieve
424 427 to the end of the session.
425 428 raw : bool
426 429 If True, return untranslated input
427 430 output : bool
428 431 If True, attempt to include output. This will be 'real' Python
429 432 objects for the current session, or text reprs from previous
430 433 sessions if db_log_output was enabled at the time. Where no output
431 434 is found, None is used.
432 435
433 436 Returns
434 437 -------
435 438 entries
436 439 An iterator over the desired lines. Each line is a 3-tuple, either
437 440 (session, line, input) if output is False, or
438 441 (session, line, (input, output)) if output is True.
439 442 """
440 443 if stop:
441 444 lineclause = "line >= ? AND line < ?"
442 445 params = (session, start, stop)
443 446 else:
444 447 lineclause = "line>=?"
445 448 params = (session, start)
446 449
447 450 return self._run_sql("WHERE session==? AND %s" % lineclause,
448 451 params, raw=raw, output=output)
449 452
450 453 def get_range_by_str(self, rangestr, raw=True, output=False):
451 454 """Get lines of history from a string of ranges, as used by magic
452 455 commands %hist, %save, %macro, etc.
453 456
454 457 Parameters
455 458 ----------
456 459 rangestr : str
457 460 A string specifying ranges, e.g. "5 ~2/1-4". See
458 461 :func:`magic_history` for full details.
459 462 raw, output : bool
460 463 As :meth:`get_range`
461 464
462 465 Returns
463 466 -------
464 467 Tuples as :meth:`get_range`
465 468 """
466 469 for sess, s, e in extract_hist_ranges(rangestr):
467 470 for line in self.get_range(sess, s, e, raw=raw, output=output):
468 471 yield line
469 472
470 473
471 474 class HistoryManager(HistoryAccessor):
472 475 """A class to organize all history-related functionality in one place.
473 476 """
474 477 # Public interface
475 478
476 479 # An instance of the IPython shell we are attached to
477 480 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
478 481 allow_none=True)
479 482 # Lists to hold processed and raw history. These start with a blank entry
480 483 # so that we can index them starting from 1
481 484 input_hist_parsed = List([""])
482 485 input_hist_raw = List([""])
483 486 # A list of directories visited during session
484 487 dir_hist = List()
485 488 @default('dir_hist')
486 489 def _dir_hist_default(self):
487 490 try:
488 491 return [py3compat.getcwd()]
489 492 except OSError:
490 493 return []
491 494
492 495 # A dict of output history, keyed with ints from the shell's
493 496 # execution count.
494 497 output_hist = Dict()
495 498 # The text/plain repr of outputs.
496 499 output_hist_reprs = Dict()
497 500
498 501 # The number of the current session in the history database
499 502 session_number = Integer()
500 503
501 504 db_log_output = Bool(False,
502 505 help="Should the history database include output? (default: no)"
503 506 ).tag(config=True)
504 507 db_cache_size = Integer(0,
505 508 help="Write to database every x commands (higher values save disk access & power).\n"
506 509 "Values of 1 or less effectively disable caching."
507 510 ).tag(config=True)
508 511 # The input and output caches
509 512 db_input_cache = List()
510 513 db_output_cache = List()
511 514
512 515 # History saving in separate thread
513 516 save_thread = Instance('IPython.core.history.HistorySavingThread',
514 517 allow_none=True)
515 518 try: # Event is a function returning an instance of _Event...
516 519 save_flag = Instance(threading._Event, allow_none=True)
517 520 except AttributeError: # ...until Python 3.3, when it's a class.
518 521 save_flag = Instance(threading.Event, allow_none=True)
519 522
520 523 # Private interface
521 524 # Variables used to store the three last inputs from the user. On each new
522 525 # history update, we populate the user's namespace with these, shifted as
523 526 # necessary.
524 527 _i00 = Unicode(u'')
525 528 _i = Unicode(u'')
526 529 _ii = Unicode(u'')
527 530 _iii = Unicode(u'')
528 531
529 532 # A regex matching all forms of the exit command, so that we don't store
530 533 # them in the history (it's annoying to rewind the first entry and land on
531 534 # an exit call).
532 535 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
533 536
534 537 def __init__(self, shell=None, config=None, **traits):
535 538 """Create a new history manager associated with a shell instance.
536 539 """
537 540 # We need a pointer back to the shell for various tasks.
538 541 super(HistoryManager, self).__init__(shell=shell, config=config,
539 542 **traits)
540 543 self.save_flag = threading.Event()
541 544 self.db_input_cache_lock = threading.Lock()
542 545 self.db_output_cache_lock = threading.Lock()
543 546
544 547 try:
545 548 self.new_session()
546 549 except OperationalError:
547 550 self.log.error("Failed to create history session in %s. History will not be saved.",
548 551 self.hist_file, exc_info=True)
549 552 self.hist_file = ':memory:'
550 553
551 554 if self.enabled and self.hist_file != ':memory:':
552 555 self.save_thread = HistorySavingThread(self)
553 556 self.save_thread.start()
554 557
555 558 def _get_hist_file_name(self, profile=None):
556 559 """Get default history file name based on the Shell's profile.
557 560
558 561 The profile parameter is ignored, but must exist for compatibility with
559 562 the parent class."""
560 563 profile_dir = self.shell.profile_dir.location
561 564 return os.path.join(profile_dir, 'history.sqlite')
562 565
563 566 @needs_sqlite
564 567 def new_session(self, conn=None):
565 568 """Get a new session number."""
566 569 if conn is None:
567 570 conn = self.db
568 571
569 572 with conn:
570 573 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
571 574 NULL, "") """, (datetime.datetime.now(),))
572 575 self.session_number = cur.lastrowid
573 576
574 577 def end_session(self):
575 578 """Close the database session, filling in the end time and line count."""
576 579 self.writeout_cache()
577 580 with self.db:
578 581 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
579 582 session==?""", (datetime.datetime.now(),
580 583 len(self.input_hist_parsed)-1, self.session_number))
581 584 self.session_number = 0
582 585
583 586 def name_session(self, name):
584 587 """Give the current session a name in the history database."""
585 588 with self.db:
586 589 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
587 590 (name, self.session_number))
588 591
589 592 def reset(self, new_session=True):
590 593 """Clear the session history, releasing all object references, and
591 594 optionally open a new session."""
592 595 self.output_hist.clear()
593 596 # The directory history can't be completely empty
594 597 self.dir_hist[:] = [py3compat.getcwd()]
595 598
596 599 if new_session:
597 600 if self.session_number:
598 601 self.end_session()
599 602 self.input_hist_parsed[:] = [""]
600 603 self.input_hist_raw[:] = [""]
601 604 self.new_session()
602 605
603 606 # ------------------------------
604 607 # Methods for retrieving history
605 608 # ------------------------------
606 609 def get_session_info(self, session=0):
607 610 """Get info about a session.
608 611
609 612 Parameters
610 613 ----------
611 614
612 615 session : int
613 616 Session number to retrieve. The current session is 0, and negative
614 617 numbers count back from current session, so -1 is the previous session.
615 618
616 619 Returns
617 620 -------
618 621
619 622 session_id : int
620 623 Session ID number
621 624 start : datetime
622 625 Timestamp for the start of the session.
623 626 end : datetime
624 627 Timestamp for the end of the session, or None if IPython crashed.
625 628 num_cmds : int
626 629 Number of commands run, or None if IPython crashed.
627 630 remark : unicode
628 631 A manually set description.
629 632 """
630 633 if session <= 0:
631 634 session += self.session_number
632 635
633 636 return super(HistoryManager, self).get_session_info(session=session)
634 637
635 638 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
636 639 """Get input and output history from the current session. Called by
637 640 get_range, and takes similar parameters."""
638 641 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
639 642
640 643 n = len(input_hist)
641 644 if start < 0:
642 645 start += n
643 646 if not stop or (stop > n):
644 647 stop = n
645 648 elif stop < 0:
646 649 stop += n
647 650
648 651 for i in range(start, stop):
649 652 if output:
650 653 line = (input_hist[i], self.output_hist_reprs.get(i))
651 654 else:
652 655 line = input_hist[i]
653 656 yield (0, i, line)
654 657
655 658 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
656 659 """Retrieve input by session.
657 660
658 661 Parameters
659 662 ----------
660 663 session : int
661 664 Session number to retrieve. The current session is 0, and negative
662 665 numbers count back from current session, so -1 is previous session.
663 666 start : int
664 667 First line to retrieve.
665 668 stop : int
666 669 End of line range (excluded from output itself). If None, retrieve
667 670 to the end of the session.
668 671 raw : bool
669 672 If True, return untranslated input
670 673 output : bool
671 674 If True, attempt to include output. This will be 'real' Python
672 675 objects for the current session, or text reprs from previous
673 676 sessions if db_log_output was enabled at the time. Where no output
674 677 is found, None is used.
675 678
676 679 Returns
677 680 -------
678 681 entries
679 682 An iterator over the desired lines. Each line is a 3-tuple, either
680 683 (session, line, input) if output is False, or
681 684 (session, line, (input, output)) if output is True.
682 685 """
683 686 if session <= 0:
684 687 session += self.session_number
685 688 if session==self.session_number: # Current session
686 689 return self._get_range_session(start, stop, raw, output)
687 690 return super(HistoryManager, self).get_range(session, start, stop, raw,
688 691 output)
689 692
690 693 ## ----------------------------
691 694 ## Methods for storing history:
692 695 ## ----------------------------
693 696 def store_inputs(self, line_num, source, source_raw=None):
694 697 """Store source and raw input in history and create input cache
695 698 variables ``_i*``.
696 699
697 700 Parameters
698 701 ----------
699 702 line_num : int
700 703 The prompt number of this input.
701 704
702 705 source : str
703 706 Python input.
704 707
705 708 source_raw : str, optional
706 709 If given, this is the raw input without any IPython transformations
707 710 applied to it. If not given, ``source`` is used.
708 711 """
709 712 if source_raw is None:
710 713 source_raw = source
711 714 source = source.rstrip('\n')
712 715 source_raw = source_raw.rstrip('\n')
713 716
714 717 # do not store exit/quit commands
715 718 if self._exit_re.match(source_raw.strip()):
716 719 return
717 720
718 721 self.input_hist_parsed.append(source)
719 722 self.input_hist_raw.append(source_raw)
720 723
721 724 with self.db_input_cache_lock:
722 725 self.db_input_cache.append((line_num, source, source_raw))
723 726 # Trigger to flush cache and write to DB.
724 727 if len(self.db_input_cache) >= self.db_cache_size:
725 728 self.save_flag.set()
726 729
727 730 # update the auto _i variables
728 731 self._iii = self._ii
729 732 self._ii = self._i
730 733 self._i = self._i00
731 734 self._i00 = source_raw
732 735
733 736 # hackish access to user namespace to create _i1,_i2... dynamically
734 737 new_i = '_i%s' % line_num
735 738 to_main = {'_i': self._i,
736 739 '_ii': self._ii,
737 740 '_iii': self._iii,
738 741 new_i : self._i00 }
739 742
740 743 if self.shell is not None:
741 744 self.shell.push(to_main, interactive=False)
742 745
743 746 def store_output(self, line_num):
744 747 """If database output logging is enabled, this saves all the
745 748 outputs from the indicated prompt number to the database. It's
746 749 called by run_cell after code has been executed.
747 750
748 751 Parameters
749 752 ----------
750 753 line_num : int
751 754 The line number from which to save outputs
752 755 """
753 756 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
754 757 return
755 758 output = self.output_hist_reprs[line_num]
756 759
757 760 with self.db_output_cache_lock:
758 761 self.db_output_cache.append((line_num, output))
759 762 if self.db_cache_size <= 1:
760 763 self.save_flag.set()
761 764
762 765 def _writeout_input_cache(self, conn):
763 766 with conn:
764 767 for line in self.db_input_cache:
765 768 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
766 769 (self.session_number,)+line)
767 770
768 771 def _writeout_output_cache(self, conn):
769 772 with conn:
770 773 for line in self.db_output_cache:
771 774 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
772 775 (self.session_number,)+line)
773 776
774 777 @needs_sqlite
775 778 def writeout_cache(self, conn=None):
776 779 """Write any entries in the cache to the database."""
777 780 if conn is None:
778 781 conn = self.db
779 782
780 783 with self.db_input_cache_lock:
781 784 try:
782 785 self._writeout_input_cache(conn)
783 786 except sqlite3.IntegrityError:
784 787 self.new_session(conn)
785 788 print("ERROR! Session/line number was not unique in",
786 789 "database. History logging moved to new session",
787 790 self.session_number)
788 791 try:
789 792 # Try writing to the new session. If this fails, don't
790 793 # recurse
791 794 self._writeout_input_cache(conn)
792 795 except sqlite3.IntegrityError:
793 796 pass
794 797 finally:
795 798 self.db_input_cache = []
796 799
797 800 with self.db_output_cache_lock:
798 801 try:
799 802 self._writeout_output_cache(conn)
800 803 except sqlite3.IntegrityError:
801 804 print("!! Session/line number for output was not unique",
802 805 "in database. Output will not be stored.")
803 806 finally:
804 807 self.db_output_cache = []
805 808
806 809
807 810 class HistorySavingThread(threading.Thread):
808 811 """This thread takes care of writing history to the database, so that
809 812 the UI isn't held up while that happens.
810 813
811 814 It waits for the HistoryManager's save_flag to be set, then writes out
812 815 the history cache. The main thread is responsible for setting the flag when
813 816 the cache size reaches a defined threshold."""
814 817 daemon = True
815 818 stop_now = False
816 819 enabled = True
817 820 def __init__(self, history_manager):
818 821 super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
819 822 self.history_manager = history_manager
820 823 self.enabled = history_manager.enabled
821 824 atexit.register(self.stop)
822 825
823 826 @needs_sqlite
824 827 def run(self):
825 828 # We need a separate db connection per thread:
826 829 try:
827 830 self.db = sqlite3.connect(self.history_manager.hist_file,
828 831 **self.history_manager.connection_options
829 832 )
830 833 while True:
831 834 self.history_manager.save_flag.wait()
832 835 if self.stop_now:
833 836 self.db.close()
834 837 return
835 838 self.history_manager.save_flag.clear()
836 839 self.history_manager.writeout_cache(self.db)
837 840 except Exception as e:
838 841 print(("The history saving thread hit an unexpected error (%s)."
839 842 "History will not be written to the database.") % repr(e))
840 843
841 844 def stop(self):
842 845 """This can be called from the main thread to safely stop this thread.
843 846
844 847 Note that it does not attempt to write out remaining history before
845 848 exiting. That should be done by calling the HistoryManager's
846 849 end_session method."""
847 850 self.stop_now = True
848 851 self.history_manager.save_flag.set()
849 852 self.join()
850 853
851 854
852 855 # To match, e.g. ~5/8-~2/3
853 856 range_re = re.compile(r"""
854 857 ((?P<startsess>~?\d+)/)?
855 858 (?P<start>\d+)?
856 859 ((?P<sep>[\-:])
857 860 ((?P<endsess>~?\d+)/)?
858 861 (?P<end>\d+))?
859 862 $""", re.VERBOSE)
860 863
861 864
862 865 def extract_hist_ranges(ranges_str):
863 866 """Turn a string of history ranges into 3-tuples of (session, start, stop).
864 867
865 868 Examples
866 869 --------
867 870 >>> list(extract_hist_ranges("~8/5-~7/4 2"))
868 871 [(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
869 872 """
870 873 for range_str in ranges_str.split():
871 874 rmatch = range_re.match(range_str)
872 875 if not rmatch:
873 876 continue
874 877 start = rmatch.group("start")
875 878 if start:
876 879 start = int(start)
877 880 end = rmatch.group("end")
878 881 # If no end specified, get (a, a + 1)
879 882 end = int(end) if end else start + 1
880 883 else: # start not specified
881 884 if not rmatch.group('startsess'): # no startsess
882 885 continue
883 886 start = 1
884 887 end = None # provide the entire session hist
885 888
886 889 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
887 890 end += 1
888 891 startsess = rmatch.group("startsess") or "0"
889 892 endsess = rmatch.group("endsess") or startsess
890 893 startsess = int(startsess.replace("~","-"))
891 894 endsess = int(endsess.replace("~","-"))
892 895 assert endsess >= startsess, "start session must be earlier than end session"
893 896
894 897 if endsess == startsess:
895 898 yield (startsess, start, end)
896 899 continue
897 900 # Multiple sessions in one range:
898 901 yield (startsess, start, None)
899 902 for sess in range(startsess+1, endsess):
900 903 yield (sess, 1, None)
901 904 yield (endsess, 1, end)
902 905
903 906
904 907 def _format_lineno(session, line):
905 908 """Helper function to format line numbers properly."""
906 909 if session == 0:
907 910 return str(line)
908 911 return "%s#%s" % (session, line)
General Comments 0
You need to be logged in to leave comments. Login now