##// END OF EJS Templates
Update history.py to use pathlib...
Jakub Klus -
r26129:948a79c2
parent child
Show More
@@ -6,7 +6,7
6 6
7 7 import atexit
8 8 import datetime
9 import os
9 from pathlib import Path
10 10 import re
11 11 import sqlite3
12 12 import threading
@@ -16,8 +16,8 from decorator import decorator
16 16 from IPython.utils.decorators import undoc
17 17 from IPython.paths import locate_profile
18 18 from traitlets import (
19 Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
20 default, observe,
19 Any, Bool, Dict, Instance, Integer, List, Unicode, Union, TraitError,
20 default, observe
21 21 )
22 22
23 23 #-----------------------------------------------------------------------------
@@ -27,17 +27,17 from traitlets import (
27 27 @undoc
28 28 class DummyDB(object):
29 29 """Dummy DB that will act as a black hole for history.
30
30
31 31 Only used in the absence of sqlite"""
32 32 def execute(*args, **kwargs):
33 33 return []
34
34
35 35 def commit(self, *args, **kwargs):
36 36 pass
37
37
38 38 def __enter__(self, *args, **kwargs):
39 39 pass
40
40
41 41 def __exit__(self, *args, **kwargs):
42 42 pass
43 43
@@ -73,17 +73,18 def catch_corrupt_db(f, self, *a, **kw):
73 73 if self._corrupt_db_counter > self._corrupt_db_limit:
74 74 self.hist_file = ':memory:'
75 75 self.log.error("Failed to load history too many times, history will not be saved.")
76 elif os.path.isfile(self.hist_file):
76 elif self.hist_file.is_file():
77 77 # move the file out of the way
78 base, ext = os.path.splitext(self.hist_file)
79 size = os.stat(self.hist_file).st_size
78 base = str(self.hist_file.parent / self.hist_file.stem)
79 ext = self.hist_file.suffix
80 size = self.hist_file.stat().st_size
80 81 if size >= _SAVE_DB_SIZE:
81 82 # if there's significant content, avoid clobbering
82 83 now = datetime.datetime.now().isoformat().replace(':', '.')
83 84 newpath = base + '-corrupt-' + now + ext
84 85 # don't clobber previous corrupt backups
85 86 for i in range(100):
86 if not os.path.isfile(newpath):
87 if not Path(newpath).exists():
87 88 break
88 89 else:
89 90 newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
@@ -91,14 +92,14 def catch_corrupt_db(f, self, *a, **kw):
91 92 # not much content, possibly empty; don't worry about clobbering
92 93 # maybe we should just delete it?
93 94 newpath = base + '-corrupt' + ext
94 os.rename(self.hist_file, newpath)
95 self.hist_file.rename(newpath)
95 96 self.log.error("History file was moved to %s and a new file created.", newpath)
96 97 self.init_db()
97 98 return []
98 99 else:
99 100 # Failed with :memory:, something serious is wrong
100 101 raise
101
102
102 103 class HistoryAccessorBase(LoggingConfigurable):
103 104 """An abstract class for History Accessors """
104 105
@@ -118,7 +119,7 class HistoryAccessorBase(LoggingConfigurable):
118 119
119 120 class HistoryAccessor(HistoryAccessorBase):
120 121 """Access the history database without adding to it.
121
122
122 123 This is intended for use by standalone history tools. IPython shells use
123 124 HistoryManager, below, which is a subclass of this."""
124 125
@@ -128,37 +129,38 class HistoryAccessor(HistoryAccessorBase):
128 129 _corrupt_db_limit = 2
129 130
130 131 # String holding the path to the history file
131 hist_file = Unicode(
132 hist_file = Union([Instance(Path), Unicode()],
132 133 help="""Path to file to use for SQLite history database.
133
134
134 135 By default, IPython will put the history database in the IPython
135 136 profile directory. If you would rather share one history among
136 137 profiles, you can set this value in each, so that they are consistent.
137
138
138 139 Due to an issue with fcntl, SQLite is known to misbehave on some NFS
139 140 mounts. If you see IPython hanging, try setting this to something on a
140 141 local disk, e.g::
141
142
142 143 ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
143 144
144 145 you can also use the specific value `:memory:` (including the colon
145 146 at both end but not the back ticks), to avoid creating an history file.
146
147 """).tag(config=True)
148
147
148 """
149 ).tag(config=True)
150
149 151 enabled = Bool(True,
150 152 help="""enable the SQLite history
151
153
152 154 set enabled=False to disable the SQLite history,
153 155 in which case there will be no stored history, no SQLite connection,
154 156 and no background saving thread. This may be necessary in some
155 157 threaded environments where IPython is embedded.
156 158 """
157 159 ).tag(config=True)
158
160
159 161 connection_options = Dict(
160 162 help="""Options for configuring the SQLite connection
161
163
162 164 These options are passed as keyword args to sqlite3.connect
163 165 when establishing database connections.
164 166 """
@@ -175,10 +177,10 class HistoryAccessor(HistoryAccessorBase):
175 177 msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
176 178 (self.__class__.__name__, new)
177 179 raise TraitError(msg)
178
179 def __init__(self, profile='default', hist_file=u'', **traits):
180
181 def __init__(self, profile='default', hist_file="", **traits):
180 182 """Create a new history accessor.
181
183
182 184 Parameters
183 185 ----------
184 186 profile : str
@@ -196,33 +198,35 class HistoryAccessor(HistoryAccessorBase):
196 198 # set by config
197 199 if hist_file:
198 200 self.hist_file = hist_file
199
200 if self.hist_file == u'':
201
202 try:
203 self.hist_file
204 except TraitError:
201 205 # No one has set the hist_file, yet.
202 206 self.hist_file = self._get_hist_file_name(profile)
203
207
204 208 self.init_db()
205
209
206 210 def _get_hist_file_name(self, profile='default'):
207 211 """Find the history file for the given profile name.
208
212
209 213 This is overridden by the HistoryManager subclass, to use the shell's
210 214 active profile.
211
215
212 216 Parameters
213 217 ----------
214 218 profile : str
215 219 The name of a profile which has a history file.
216 220 """
217 return os.path.join(locate_profile(profile), 'history.sqlite')
218
221 return Path(locate_profile(profile)) / 'history.sqlite'
222
219 223 @catch_corrupt_db
220 224 def init_db(self):
221 225 """Connect to the database, and create tables if necessary."""
222 226 if not self.enabled:
223 227 self.db = DummyDB()
224 228 return
225
229
226 230 # use detect_types so that timestamps return datetime objects
227 231 kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
228 232 kwargs.update(self.connection_options)
@@ -290,7 +294,7 class HistoryAccessor(HistoryAccessorBase):
290 294
291 295 Returns
292 296 -------
293
297
294 298 session_id : int
295 299 Session ID number
296 300 start : datetime
@@ -308,7 +312,7 class HistoryAccessor(HistoryAccessorBase):
308 312 @catch_corrupt_db
309 313 def get_last_session_id(self):
310 314 """Get the last session ID currently in the database.
311
315
312 316 Within IPython, this should be the same as the value stored in
313 317 :attr:`HistoryManager.session_number`.
314 318 """
@@ -384,7 +388,7 class HistoryAccessor(HistoryAccessorBase):
384 388 if n is not None:
385 389 return reversed(list(cur))
386 390 return cur
387
391
388 392 @catch_corrupt_db
389 393 def get_range(self, session, start=1, stop=None, raw=True,output=False):
390 394 """Retrieve input by session.
@@ -461,7 +465,7 class HistoryManager(HistoryAccessor):
461 465 @default('dir_hist')
462 466 def _dir_hist_default(self):
463 467 try:
464 return [os.getcwd()]
468 return [Path.cwd()]
465 469 except OSError:
466 470 return []
467 471
@@ -473,7 +477,7 class HistoryManager(HistoryAccessor):
473 477
474 478 # The number of the current session in the history database
475 479 session_number = Integer()
476
480
477 481 db_log_output = Bool(False,
478 482 help="Should the history database include output? (default: no)"
479 483 ).tag(config=True)
@@ -484,12 +488,12 class HistoryManager(HistoryAccessor):
484 488 # The input and output caches
485 489 db_input_cache = List()
486 490 db_output_cache = List()
487
491
488 492 # History saving in separate thread
489 493 save_thread = Instance('IPython.core.history.HistorySavingThread',
490 494 allow_none=True)
491 495 save_flag = Instance(threading.Event, allow_none=True)
492
496
493 497 # Private interface
494 498 # Variables used to store the three last inputs from the user. On each new
495 499 # history update, we populate the user's namespace with these, shifted as
@@ -513,37 +517,37 class HistoryManager(HistoryAccessor):
513 517 self.save_flag = threading.Event()
514 518 self.db_input_cache_lock = threading.Lock()
515 519 self.db_output_cache_lock = threading.Lock()
516
520
517 521 try:
518 522 self.new_session()
519 523 except sqlite3.OperationalError:
520 524 self.log.error("Failed to create history session in %s. History will not be saved.",
521 525 self.hist_file, exc_info=True)
522 526 self.hist_file = ':memory:'
523
527
524 528 if self.enabled and self.hist_file != ':memory:':
525 529 self.save_thread = HistorySavingThread(self)
526 530 self.save_thread.start()
527 531
528 532 def _get_hist_file_name(self, profile=None):
529 533 """Get default history file name based on the Shell's profile.
530
534
531 535 The profile parameter is ignored, but must exist for compatibility with
532 536 the parent class."""
533 537 profile_dir = self.shell.profile_dir.location
534 return os.path.join(profile_dir, 'history.sqlite')
535
538 return Path(profile_dir)/'history.sqlite'
539
536 540 @only_when_enabled
537 541 def new_session(self, conn=None):
538 542 """Get a new session number."""
539 543 if conn is None:
540 544 conn = self.db
541
545
542 546 with conn:
543 547 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
544 548 NULL, "") """, (datetime.datetime.now(),))
545 549 self.session_number = cur.lastrowid
546
550
547 551 def end_session(self):
548 552 """Close the database session, filling in the end time and line count."""
549 553 self.writeout_cache()
@@ -552,20 +556,20 class HistoryManager(HistoryAccessor):
552 556 session==?""", (datetime.datetime.now(),
553 557 len(self.input_hist_parsed)-1, self.session_number))
554 558 self.session_number = 0
555
559
556 560 def name_session(self, name):
557 561 """Give the current session a name in the history database."""
558 562 with self.db:
559 563 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
560 564 (name, self.session_number))
561
565
562 566 def reset(self, new_session=True):
563 567 """Clear the session history, releasing all object references, and
564 568 optionally open a new session."""
565 569 self.output_hist.clear()
566 570 # The directory history can't be completely empty
567 self.dir_hist[:] = [os.getcwd()]
568
571 self.dir_hist[:] = [Path.cwd()]
572
569 573 if new_session:
570 574 if self.session_number:
571 575 self.end_session()
@@ -588,7 +592,7 class HistoryManager(HistoryAccessor):
588 592
589 593 Returns
590 594 -------
591
595
592 596 session_id : int
593 597 Session ID number
594 598 start : datetime
@@ -609,7 +613,7 class HistoryManager(HistoryAccessor):
609 613 """Get input and output history from the current session. Called by
610 614 get_range, and takes similar parameters."""
611 615 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
612
616
613 617 n = len(input_hist)
614 618 if start < 0:
615 619 start += n
@@ -617,17 +621,17 class HistoryManager(HistoryAccessor):
617 621 stop = n
618 622 elif stop < 0:
619 623 stop += n
620
624
621 625 for i in range(start, stop):
622 626 if output:
623 627 line = (input_hist[i], self.output_hist_reprs.get(i))
624 628 else:
625 629 line = input_hist[i]
626 630 yield (0, i, line)
627
631
628 632 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
629 633 """Retrieve input by session.
630
634
631 635 Parameters
632 636 ----------
633 637 session : int
@@ -645,7 +649,7 class HistoryManager(HistoryAccessor):
645 649 objects for the current session, or text reprs from previous
646 650 sessions if db_log_output was enabled at the time. Where no output
647 651 is found, None is used.
648
652
649 653 Returns
650 654 -------
651 655 entries
@@ -709,7 +713,7 class HistoryManager(HistoryAccessor):
709 713 '_ii': self._ii,
710 714 '_iii': self._iii,
711 715 new_i : self._i00 }
712
716
713 717 if self.shell is not None:
714 718 self.shell.push(to_main, interactive=False)
715 719
@@ -7,7 +7,7
7 7
8 8 # stdlib
9 9 import io
10 import os
10 from pathlib import Path
11 11 import sys
12 12 import tempfile
13 13 from datetime import datetime
@@ -29,8 +29,9 def test_proper_default_encoding():
29 29 def test_history():
30 30 ip = get_ipython()
31 31 with TemporaryDirectory() as tmpdir:
32 tmp_path = Path(tmpdir)
32 33 hist_manager_ori = ip.history_manager
33 hist_file = os.path.join(tmpdir, 'history.sqlite')
34 hist_file = tmp_path / 'history.sqlite'
34 35 try:
35 36 ip.history_manager = HistoryManager(shell=ip, hist_file=hist_file)
36 37 hist = [u'a=1', u'def f():\n test = 1\n return test', u"b='€Æ¾÷ß'"]
@@ -55,10 +56,10 def test_history():
55 56 ip.magic('%hist 2-500')
56 57
57 58 # Check that we can write non-ascii characters to a file
58 ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1"))
59 ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2"))
60 ip.magic("%%hist -nf %s" % os.path.join(tmpdir, "test3"))
61 ip.magic("%%save %s 1-10" % os.path.join(tmpdir, "test4"))
59 ip.magic("%%hist -f %s" % (tmp_path / "test1"))
60 ip.magic("%%hist -pf %s" % (tmp_path / "test2"))
61 ip.magic("%%hist -nf %s" % (tmp_path / "test3"))
62 ip.magic("%%save %s 1-10" % (tmp_path / "test4"))
62 63
63 64 # New session
64 65 ip.history_manager.reset()
@@ -126,8 +127,8 def test_history():
126 127 nt.assert_equal(list(gothist), [(1,3,(hist[2],"spam"))] )
127 128
128 129 # Cross testing: check that magic %save can get previous session.
129 testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
130 ip.magic("save " + testfilename + " ~1/1-3")
130 testfilename = (tmp_path /"test.py").resolve()
131 ip.magic("save " + str(testfilename) + " ~1/1-3")
131 132 with io.open(testfilename, encoding='utf-8') as testfile:
132 133 nt.assert_equal(testfile.read(),
133 134 u"# coding: utf-8\n" + u"\n".join(hist)+u"\n")
@@ -176,13 +177,13 def test_timestamp_type():
176 177 def test_hist_file_config():
177 178 cfg = Config()
178 179 tfile = tempfile.NamedTemporaryFile(delete=False)
179 cfg.HistoryManager.hist_file = tfile.name
180 cfg.HistoryManager.hist_file = Path(tfile.name)
180 181 try:
181 182 hm = HistoryManager(shell=get_ipython(), config=cfg)
182 183 nt.assert_equal(hm.hist_file, cfg.HistoryManager.hist_file)
183 184 finally:
184 185 try:
185 os.remove(tfile.name)
186 Path(tfile.name).unlink()
186 187 except OSError:
187 188 # same catch as in testing.tools.TempFileMixin
188 189 # On Windows, even though we close the file, we still can't
@@ -197,7 +198,7 def test_histmanager_disabled():
197 198 ip = get_ipython()
198 199 with TemporaryDirectory() as tmpdir:
199 200 hist_manager_ori = ip.history_manager
200 hist_file = os.path.join(tmpdir, 'history.sqlite')
201 hist_file = Path(tmpdir) / 'history.sqlite'
201 202 cfg.HistoryManager.hist_file = hist_file
202 203 try:
203 204 ip.history_manager = HistoryManager(shell=ip, config=cfg)
@@ -211,4 +212,4 def test_histmanager_disabled():
211 212 ip.history_manager = hist_manager_ori
212 213
213 214 # hist_file should not be created
214 nt.assert_false(os.path.exists(hist_file))
215 nt.assert_false(hist_file.exists())
@@ -10,6 +10,7 Authors
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 import os
13 from pathlib import Path
13 14 import re
14 15 import sys
15 16 import tempfile
@@ -142,7 +143,7 def default_config():
142 143 config.TerminalTerminalInteractiveShell.term_title = False,
143 144 config.TerminalInteractiveShell.autocall = 0
144 145 f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
145 config.HistoryManager.hist_file = f.name
146 config.HistoryManager.hist_file = Path(f.name)
146 147 f.close()
147 148 config.HistoryManager.db_cache_size = 10000
148 149 return config
General Comments 0
You need to be logged in to leave comments. Login now