##// END OF EJS Templates
Move with statement so history db cache is written out in one transaction (two if output history is being logged).
Thomas Kluyver -
Show More
@@ -1,748 +1,748 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2 #-----------------------------------------------------------------------------
2 #-----------------------------------------------------------------------------
3 # Copyright (C) 2010 The IPython Development Team.
3 # Copyright (C) 2010 The IPython Development Team.
4 #
4 #
5 # Distributed under the terms of the BSD License.
5 # Distributed under the terms of the BSD License.
6 #
6 #
7 # The full license is in the file COPYING.txt, distributed with this software.
7 # The full license is in the file COPYING.txt, distributed with this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 from __future__ import print_function
13 from __future__ import print_function
14
14
15 # Stdlib imports
15 # Stdlib imports
16 import datetime
16 import datetime
17 import json
17 import json
18 import os
18 import os
19 import re
19 import re
20 import sqlite3
20 import sqlite3
21
21
22 from collections import defaultdict
22 from collections import defaultdict
23
23
24 # Our own packages
24 # Our own packages
25 from IPython.config.configurable import Configurable
25 from IPython.config.configurable import Configurable
26 import IPython.utils.io
26 import IPython.utils.io
27
27
28 from IPython.testing import decorators as testdec
28 from IPython.testing import decorators as testdec
29 from IPython.utils.io import ask_yes_no
29 from IPython.utils.io import ask_yes_no
30 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
30 from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
31 from IPython.utils.warn import warn
31 from IPython.utils.warn import warn
32
32
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34 # Classes and functions
34 # Classes and functions
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36
36
37 class HistoryManager(Configurable):
37 class HistoryManager(Configurable):
38 """A class to organize all history-related functionality in one place.
38 """A class to organize all history-related functionality in one place.
39 """
39 """
40 # Public interface
40 # Public interface
41
41
42 # An instance of the IPython shell we are attached to
42 # An instance of the IPython shell we are attached to
43 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
43 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
44 # Lists to hold processed and raw history. These start with a blank entry
44 # Lists to hold processed and raw history. These start with a blank entry
45 # so that we can index them starting from 1
45 # so that we can index them starting from 1
46 input_hist_parsed = List([""])
46 input_hist_parsed = List([""])
47 input_hist_raw = List([""])
47 input_hist_raw = List([""])
48 # A list of directories visited during session
48 # A list of directories visited during session
49 dir_hist = List()
49 dir_hist = List()
50 def _dir_hist_default(self):
50 def _dir_hist_default(self):
51 try:
51 try:
52 return [os.getcwd()]
52 return [os.getcwd()]
53 except OSError:
53 except OSError:
54 return []
54 return []
55
55
56 # A dict of output history, keyed with ints from the shell's
56 # A dict of output history, keyed with ints from the shell's
57 # execution count. If there are several outputs from one command,
57 # execution count. If there are several outputs from one command,
58 # only the last one is stored.
58 # only the last one is stored.
59 output_hist = Dict()
59 output_hist = Dict()
60 # Contains all outputs, in lists of reprs.
60 # Contains all outputs, in lists of reprs.
61 output_hist_reprs = Instance(defaultdict, args=(list,))
61 output_hist_reprs = Instance(defaultdict, args=(list,))
62
62
63 # String holding the path to the history file
63 # String holding the path to the history file
64 hist_file = Unicode(config=True)
64 hist_file = Unicode(config=True)
65
65
66 # The SQLite database
66 # The SQLite database
67 db = Instance(sqlite3.Connection)
67 db = Instance(sqlite3.Connection)
68 # The number of the current session in the history database
68 # The number of the current session in the history database
69 session_number = Int()
69 session_number = Int()
70 # Should we log output to the database? (default no)
70 # Should we log output to the database? (default no)
71 db_log_output = Bool(False, config=True)
71 db_log_output = Bool(False, config=True)
72 # Write to database every x commands (higher values save disk access & power)
72 # Write to database every x commands (higher values save disk access & power)
73 # Values of 1 or less effectively disable caching.
73 # Values of 1 or less effectively disable caching.
74 db_cache_size = Int(0, config=True)
74 db_cache_size = Int(0, config=True)
75 # The input and output caches
75 # The input and output caches
76 db_input_cache = List()
76 db_input_cache = List()
77 db_output_cache = List()
77 db_output_cache = List()
78
78
79 # Private interface
79 # Private interface
80 # Variables used to store the three last inputs from the user. On each new
80 # Variables used to store the three last inputs from the user. On each new
81 # history update, we populate the user's namespace with these, shifted as
81 # history update, we populate the user's namespace with these, shifted as
82 # necessary.
82 # necessary.
83 _i00 = Unicode(u'')
83 _i00 = Unicode(u'')
84 _i = Unicode(u'')
84 _i = Unicode(u'')
85 _ii = Unicode(u'')
85 _ii = Unicode(u'')
86 _iii = Unicode(u'')
86 _iii = Unicode(u'')
87
87
88 # A set with all forms of the exit command, so that we don't store them in
88 # A set with all forms of the exit command, so that we don't store them in
89 # the history (it's annoying to rewind the first entry and land on an exit
89 # the history (it's annoying to rewind the first entry and land on an exit
90 # call).
90 # call).
91 _exit_commands = Instance(set, args=(['Quit', 'quit', 'Exit', 'exit',
91 _exit_commands = Instance(set, args=(['Quit', 'quit', 'Exit', 'exit',
92 '%Quit', '%quit', '%Exit', '%exit'],))
92 '%Quit', '%quit', '%Exit', '%exit'],))
93
93
94 def __init__(self, shell, config=None, **traits):
94 def __init__(self, shell, config=None, **traits):
95 """Create a new history manager associated with a shell instance.
95 """Create a new history manager associated with a shell instance.
96 """
96 """
97 # We need a pointer back to the shell for various tasks.
97 # We need a pointer back to the shell for various tasks.
98 super(HistoryManager, self).__init__(shell=shell, config=config,
98 super(HistoryManager, self).__init__(shell=shell, config=config,
99 **traits)
99 **traits)
100
100
101 if self.hist_file == u'':
101 if self.hist_file == u'':
102 # No one has set the hist_file, yet.
102 # No one has set the hist_file, yet.
103 if shell.profile:
103 if shell.profile:
104 histfname = 'history-%s' % shell.profile
104 histfname = 'history-%s' % shell.profile
105 else:
105 else:
106 histfname = 'history'
106 histfname = 'history'
107 self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
107 self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
108
108
109 try:
109 try:
110 self.init_db()
110 self.init_db()
111 except sqlite3.DatabaseError:
111 except sqlite3.DatabaseError:
112 if os.path.isfile(self.hist_file):
112 if os.path.isfile(self.hist_file):
113 # Try to move the file out of the way.
113 # Try to move the file out of the way.
114 newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
114 newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
115 os.rename(self.hist_file, newpath)
115 os.rename(self.hist_file, newpath)
116 print("ERROR! History file wasn't a valid SQLite database.",
116 print("ERROR! History file wasn't a valid SQLite database.",
117 "It was moved to %s" % newpath, "and a new file created.")
117 "It was moved to %s" % newpath, "and a new file created.")
118 self.init_db()
118 self.init_db()
119 else:
119 else:
120 # The hist_file is probably :memory: or something else.
120 # The hist_file is probably :memory: or something else.
121 raise
121 raise
122
122
123 self.new_session()
123 self.new_session()
124
124
125
125
126 def init_db(self):
126 def init_db(self):
127 """Connect to the database, and create tables if necessary."""
127 """Connect to the database, and create tables if necessary."""
128 self.db = sqlite3.connect(self.hist_file)
128 self.db = sqlite3.connect(self.hist_file)
129 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
129 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
130 primary key autoincrement, start timestamp,
130 primary key autoincrement, start timestamp,
131 end timestamp, num_cmds integer, remark text)""")
131 end timestamp, num_cmds integer, remark text)""")
132 self.db.execute("""CREATE TABLE IF NOT EXISTS history
132 self.db.execute("""CREATE TABLE IF NOT EXISTS history
133 (session integer, line integer, source text, source_raw text,
133 (session integer, line integer, source text, source_raw text,
134 PRIMARY KEY (session, line))""")
134 PRIMARY KEY (session, line))""")
135 # Output history is optional, but ensure the table's there so it can be
135 # Output history is optional, but ensure the table's there so it can be
136 # enabled later.
136 # enabled later.
137 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
137 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
138 (session integer, line integer, output text,
138 (session integer, line integer, output text,
139 PRIMARY KEY (session, line))""")
139 PRIMARY KEY (session, line))""")
140 self.db.commit()
140 self.db.commit()
141
141
142 def new_session(self):
142 def new_session(self):
143 """Get a new session number."""
143 """Get a new session number."""
144 with self.db:
144 with self.db:
145 cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
145 cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
146 NULL, "") """, (datetime.datetime.now(),))
146 NULL, "") """, (datetime.datetime.now(),))
147 self.session_number = cur.lastrowid
147 self.session_number = cur.lastrowid
148
148
149 def end_session(self):
149 def end_session(self):
150 """Close the database session, filling in the end time and line count."""
150 """Close the database session, filling in the end time and line count."""
151 self.writeout_cache()
151 self.writeout_cache()
152 with self.db:
152 with self.db:
153 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
153 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
154 session==?""", (datetime.datetime.now(),
154 session==?""", (datetime.datetime.now(),
155 len(self.input_hist_parsed)-1, self.session_number))
155 len(self.input_hist_parsed)-1, self.session_number))
156 self.session_number = 0
156 self.session_number = 0
157
157
158 def name_session(self, name):
158 def name_session(self, name):
159 """Give the current session a name in the history database."""
159 """Give the current session a name in the history database."""
160 with self.db:
160 with self.db:
161 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
161 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
162 (name, self.session_number))
162 (name, self.session_number))
163
163
164 def reset(self, new_session=True):
164 def reset(self, new_session=True):
165 """Clear the session history, releasing all object references, and
165 """Clear the session history, releasing all object references, and
166 optionally open a new session."""
166 optionally open a new session."""
167 if self.session_number:
167 if self.session_number:
168 self.end_session()
168 self.end_session()
169 self.input_hist_parsed[:] = [""]
169 self.input_hist_parsed[:] = [""]
170 self.input_hist_raw[:] = [""]
170 self.input_hist_raw[:] = [""]
171 self.output_hist.clear()
171 self.output_hist.clear()
172 # The directory history can't be completely empty
172 # The directory history can't be completely empty
173 self.dir_hist[:] = [os.getcwd()]
173 self.dir_hist[:] = [os.getcwd()]
174
174
175 if new_session:
175 if new_session:
176 self.new_session()
176 self.new_session()
177
177
178 ## -------------------------------
178 ## -------------------------------
179 ## Methods for retrieving history:
179 ## Methods for retrieving history:
180 ## -------------------------------
180 ## -------------------------------
181 def _run_sql(self, sql, params, raw=True, output=False):
181 def _run_sql(self, sql, params, raw=True, output=False):
182 """Prepares and runs an SQL query for the history database.
182 """Prepares and runs an SQL query for the history database.
183
183
184 Parameters
184 Parameters
185 ----------
185 ----------
186 sql : str
186 sql : str
187 Any filtering expressions to go after SELECT ... FROM ...
187 Any filtering expressions to go after SELECT ... FROM ...
188 params : tuple
188 params : tuple
189 Parameters passed to the SQL query (to replace "?")
189 Parameters passed to the SQL query (to replace "?")
190 raw, output : bool
190 raw, output : bool
191 See :meth:`get_range`
191 See :meth:`get_range`
192
192
193 Returns
193 Returns
194 -------
194 -------
195 Tuples as :meth:`get_range`
195 Tuples as :meth:`get_range`
196 """
196 """
197 toget = 'source_raw' if raw else 'source'
197 toget = 'source_raw' if raw else 'source'
198 sqlfrom = "history"
198 sqlfrom = "history"
199 if output:
199 if output:
200 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
200 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
201 toget = "history.%s, output_history.output" % toget
201 toget = "history.%s, output_history.output" % toget
202 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
202 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
203 (toget, sqlfrom) + sql, params)
203 (toget, sqlfrom) + sql, params)
204 if output: # Regroup into 3-tuples, and parse JSON
204 if output: # Regroup into 3-tuples, and parse JSON
205 loads = lambda out: json.loads(out) if out else None
205 loads = lambda out: json.loads(out) if out else None
206 return ((ses, lin, (inp, loads(out))) \
206 return ((ses, lin, (inp, loads(out))) \
207 for ses, lin, inp, out in cur)
207 for ses, lin, inp, out in cur)
208 return cur
208 return cur
209
209
210
210
211 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
211 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
212 """Get the last n lines from the history database.
212 """Get the last n lines from the history database.
213
213
214 Parameters
214 Parameters
215 ----------
215 ----------
216 n : int
216 n : int
217 The number of lines to get
217 The number of lines to get
218 raw, output : bool
218 raw, output : bool
219 See :meth:`get_range`
219 See :meth:`get_range`
220 include_latest : bool
220 include_latest : bool
221 If False (default), n+1 lines are fetched, and the latest one
221 If False (default), n+1 lines are fetched, and the latest one
222 is discarded. This is intended to be used where the function
222 is discarded. This is intended to be used where the function
223 is called by a user command, which it should not return.
223 is called by a user command, which it should not return.
224
224
225 Returns
225 Returns
226 -------
226 -------
227 Tuples as :meth:`get_range`
227 Tuples as :meth:`get_range`
228 """
228 """
229 self.writeout_cache()
229 self.writeout_cache()
230 if not include_latest:
230 if not include_latest:
231 n += 1
231 n += 1
232 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
232 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
233 (n,), raw=raw, output=output)
233 (n,), raw=raw, output=output)
234 if not include_latest:
234 if not include_latest:
235 return reversed(list(cur)[1:])
235 return reversed(list(cur)[1:])
236 return reversed(list(cur))
236 return reversed(list(cur))
237
237
238 def search(self, pattern="*", raw=True, search_raw=True,
238 def search(self, pattern="*", raw=True, search_raw=True,
239 output=False):
239 output=False):
240 """Search the database using unix glob-style matching (wildcards
240 """Search the database using unix glob-style matching (wildcards
241 * and ?).
241 * and ?).
242
242
243 Parameters
243 Parameters
244 ----------
244 ----------
245 pattern : str
245 pattern : str
246 The wildcarded pattern to match when searching
246 The wildcarded pattern to match when searching
247 search_raw : bool
247 search_raw : bool
248 If True, search the raw input, otherwise, the parsed input
248 If True, search the raw input, otherwise, the parsed input
249 raw, output : bool
249 raw, output : bool
250 See :meth:`get_range`
250 See :meth:`get_range`
251
251
252 Returns
252 Returns
253 -------
253 -------
254 Tuples as :meth:`get_range`
254 Tuples as :meth:`get_range`
255 """
255 """
256 tosearch = "source_raw" if search_raw else "source"
256 tosearch = "source_raw" if search_raw else "source"
257 if output:
257 if output:
258 tosearch = "history." + tosearch
258 tosearch = "history." + tosearch
259 self.writeout_cache()
259 self.writeout_cache()
260 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
260 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
261 raw=raw, output=output)
261 raw=raw, output=output)
262
262
263 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
263 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
264 """Get input and output history from the current session. Called by
264 """Get input and output history from the current session. Called by
265 get_range, and takes similar parameters."""
265 get_range, and takes similar parameters."""
266 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
266 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
267
267
268 n = len(input_hist)
268 n = len(input_hist)
269 if start < 0:
269 if start < 0:
270 start += n
270 start += n
271 if not stop:
271 if not stop:
272 stop = n
272 stop = n
273 elif stop < 0:
273 elif stop < 0:
274 stop += n
274 stop += n
275
275
276 for i in range(start, stop):
276 for i in range(start, stop):
277 if output:
277 if output:
278 line = (input_hist[i], self.output_hist_reprs.get(i))
278 line = (input_hist[i], self.output_hist_reprs.get(i))
279 else:
279 else:
280 line = input_hist[i]
280 line = input_hist[i]
281 yield (0, i, line)
281 yield (0, i, line)
282
282
283 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
283 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
284 """Retrieve input by session.
284 """Retrieve input by session.
285
285
286 Parameters
286 Parameters
287 ----------
287 ----------
288 session : int
288 session : int
289 Session number to retrieve. The current session is 0, and negative
289 Session number to retrieve. The current session is 0, and negative
290 numbers count back from current session, so -1 is previous session.
290 numbers count back from current session, so -1 is previous session.
291 start : int
291 start : int
292 First line to retrieve.
292 First line to retrieve.
293 stop : int
293 stop : int
294 End of line range (excluded from output itself). If None, retrieve
294 End of line range (excluded from output itself). If None, retrieve
295 to the end of the session.
295 to the end of the session.
296 raw : bool
296 raw : bool
297 If True, return untranslated input
297 If True, return untranslated input
298 output : bool
298 output : bool
299 If True, attempt to include output. This will be 'real' Python
299 If True, attempt to include output. This will be 'real' Python
300 objects for the current session, or text reprs from previous
300 objects for the current session, or text reprs from previous
301 sessions if db_log_output was enabled at the time. Where no output
301 sessions if db_log_output was enabled at the time. Where no output
302 is found, None is used.
302 is found, None is used.
303
303
304 Returns
304 Returns
305 -------
305 -------
306 An iterator over the desired lines. Each line is a 3-tuple, either
306 An iterator over the desired lines. Each line is a 3-tuple, either
307 (session, line, input) if output is False, or
307 (session, line, input) if output is False, or
308 (session, line, (input, output)) if output is True.
308 (session, line, (input, output)) if output is True.
309 """
309 """
310 if session == 0 or session==self.session_number: # Current session
310 if session == 0 or session==self.session_number: # Current session
311 return self._get_range_session(start, stop, raw, output)
311 return self._get_range_session(start, stop, raw, output)
312 if session < 0:
312 if session < 0:
313 session += self.session_number
313 session += self.session_number
314
314
315 if stop:
315 if stop:
316 lineclause = "line >= ? AND line < ?"
316 lineclause = "line >= ? AND line < ?"
317 params = (session, start, stop)
317 params = (session, start, stop)
318 else:
318 else:
319 lineclause = "line>=?"
319 lineclause = "line>=?"
320 params = (session, start)
320 params = (session, start)
321
321
322 return self._run_sql("WHERE session==? AND %s""" % lineclause,
322 return self._run_sql("WHERE session==? AND %s""" % lineclause,
323 params, raw=raw, output=output)
323 params, raw=raw, output=output)
324
324
325 def get_range_by_str(self, rangestr, raw=True, output=False):
325 def get_range_by_str(self, rangestr, raw=True, output=False):
326 """Get lines of history from a string of ranges, as used by magic
326 """Get lines of history from a string of ranges, as used by magic
327 commands %hist, %save, %macro, etc.
327 commands %hist, %save, %macro, etc.
328
328
329 Parameters
329 Parameters
330 ----------
330 ----------
331 rangestr : str
331 rangestr : str
332 A string specifying ranges, e.g. "5 ~2/1-4". See
332 A string specifying ranges, e.g. "5 ~2/1-4". See
333 :func:`magic_history` for full details.
333 :func:`magic_history` for full details.
334 raw, output : bool
334 raw, output : bool
335 As :meth:`get_range`
335 As :meth:`get_range`
336
336
337 Returns
337 Returns
338 -------
338 -------
339 Tuples as :meth:`get_range`
339 Tuples as :meth:`get_range`
340 """
340 """
341 for sess, s, e in extract_hist_ranges(rangestr):
341 for sess, s, e in extract_hist_ranges(rangestr):
342 for line in self.get_range(sess, s, e, raw=raw, output=output):
342 for line in self.get_range(sess, s, e, raw=raw, output=output):
343 yield line
343 yield line
344
344
345 ## ----------------------------
345 ## ----------------------------
346 ## Methods for storing history:
346 ## Methods for storing history:
347 ## ----------------------------
347 ## ----------------------------
348 def store_inputs(self, line_num, source, source_raw=None):
348 def store_inputs(self, line_num, source, source_raw=None):
349 """Store source and raw input in history and create input cache
349 """Store source and raw input in history and create input cache
350 variables _i*.
350 variables _i*.
351
351
352 Parameters
352 Parameters
353 ----------
353 ----------
354 line_num : int
354 line_num : int
355 The prompt number of this input.
355 The prompt number of this input.
356
356
357 source : str
357 source : str
358 Python input.
358 Python input.
359
359
360 source_raw : str, optional
360 source_raw : str, optional
361 If given, this is the raw input without any IPython transformations
361 If given, this is the raw input without any IPython transformations
362 applied to it. If not given, ``source`` is used.
362 applied to it. If not given, ``source`` is used.
363 """
363 """
364 if source_raw is None:
364 if source_raw is None:
365 source_raw = source
365 source_raw = source
366 source = source.rstrip('\n')
366 source = source.rstrip('\n')
367 source_raw = source_raw.rstrip('\n')
367 source_raw = source_raw.rstrip('\n')
368
368
369 # do not store exit/quit commands
369 # do not store exit/quit commands
370 if source_raw.strip() in self._exit_commands:
370 if source_raw.strip() in self._exit_commands:
371 return
371 return
372
372
373 self.input_hist_parsed.append(source)
373 self.input_hist_parsed.append(source)
374 self.input_hist_raw.append(source_raw)
374 self.input_hist_raw.append(source_raw)
375
375
376 self.db_input_cache.append((line_num, source, source_raw))
376 self.db_input_cache.append((line_num, source, source_raw))
377 # Trigger to flush cache and write to DB.
377 # Trigger to flush cache and write to DB.
378 if len(self.db_input_cache) >= self.db_cache_size:
378 if len(self.db_input_cache) >= self.db_cache_size:
379 self.writeout_cache()
379 self.writeout_cache()
380
380
381 # update the auto _i variables
381 # update the auto _i variables
382 self._iii = self._ii
382 self._iii = self._ii
383 self._ii = self._i
383 self._ii = self._i
384 self._i = self._i00
384 self._i = self._i00
385 self._i00 = source_raw
385 self._i00 = source_raw
386
386
387 # hackish access to user namespace to create _i1,_i2... dynamically
387 # hackish access to user namespace to create _i1,_i2... dynamically
388 new_i = '_i%s' % line_num
388 new_i = '_i%s' % line_num
389 to_main = {'_i': self._i,
389 to_main = {'_i': self._i,
390 '_ii': self._ii,
390 '_ii': self._ii,
391 '_iii': self._iii,
391 '_iii': self._iii,
392 new_i : self._i00 }
392 new_i : self._i00 }
393 self.shell.user_ns.update(to_main)
393 self.shell.user_ns.update(to_main)
394
394
395 def store_output(self, line_num):
395 def store_output(self, line_num):
396 """If database output logging is enabled, this saves all the
396 """If database output logging is enabled, this saves all the
397 outputs from the indicated prompt number to the database. It's
397 outputs from the indicated prompt number to the database. It's
398 called by run_cell after code has been executed.
398 called by run_cell after code has been executed.
399
399
400 Parameters
400 Parameters
401 ----------
401 ----------
402 line_num : int
402 line_num : int
403 The line number from which to save outputs
403 The line number from which to save outputs
404 """
404 """
405 if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
405 if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
406 return
406 return
407 output = json.dumps(self.output_hist_reprs[line_num])
407 output = json.dumps(self.output_hist_reprs[line_num])
408
408
409 self.db_output_cache.append((line_num, output))
409 self.db_output_cache.append((line_num, output))
410 if self.db_cache_size <= 1:
410 if self.db_cache_size <= 1:
411 self.writeout_cache()
411 self.writeout_cache()
412
412
413 def _writeout_input_cache(self):
413 def _writeout_input_cache(self):
414 for line in self.db_input_cache:
415 with self.db:
414 with self.db:
415 for line in self.db_input_cache:
416 self.db.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
416 self.db.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
417 (self.session_number,)+line)
417 (self.session_number,)+line)
418
418
419 def _writeout_output_cache(self):
419 def _writeout_output_cache(self):
420 for line in self.db_output_cache:
421 with self.db:
420 with self.db:
421 for line in self.db_output_cache:
422 self.db.execute("INSERT INTO output_history VALUES (?, ?, ?)",
422 self.db.execute("INSERT INTO output_history VALUES (?, ?, ?)",
423 (self.session_number,)+line)
423 (self.session_number,)+line)
424
424
425 def writeout_cache(self):
425 def writeout_cache(self):
426 """Write any entries in the cache to the database."""
426 """Write any entries in the cache to the database."""
427 try:
427 try:
428 self._writeout_input_cache()
428 self._writeout_input_cache()
429 except sqlite3.IntegrityError:
429 except sqlite3.IntegrityError:
430 self.new_session()
430 self.new_session()
431 print("ERROR! Session/line number was not unique in",
431 print("ERROR! Session/line number was not unique in",
432 "database. History logging moved to new session",
432 "database. History logging moved to new session",
433 self.session_number)
433 self.session_number)
434 try: # Try writing to the new session. If this fails, don't recurse
434 try: # Try writing to the new session. If this fails, don't recurse
435 self.writeout_cache()
435 self._writeout_input_cache()
436 except sqlite3.IntegrityError:
436 except sqlite3.IntegrityError:
437 pass
437 pass
438 finally:
438 finally:
439 self.db_input_cache = []
439 self.db_input_cache = []
440
440
441 try:
441 try:
442 self._writeout_output_cache()
442 self._writeout_output_cache()
443 except sqlite3.IntegrityError:
443 except sqlite3.IntegrityError:
444 print("!! Session/line number for output was not unique",
444 print("!! Session/line number for output was not unique",
445 "in database. Output will not be stored.")
445 "in database. Output will not be stored.")
446 finally:
446 finally:
447 self.db_output_cache = []
447 self.db_output_cache = []
448
448
449
449
450 # To match, e.g. ~5/8-~2/3
450 # To match, e.g. ~5/8-~2/3
451 range_re = re.compile(r"""
451 range_re = re.compile(r"""
452 ((?P<startsess>~?\d+)/)?
452 ((?P<startsess>~?\d+)/)?
453 (?P<start>\d+) # Only the start line num is compulsory
453 (?P<start>\d+) # Only the start line num is compulsory
454 ((?P<sep>[\-:])
454 ((?P<sep>[\-:])
455 ((?P<endsess>~?\d+)/)?
455 ((?P<endsess>~?\d+)/)?
456 (?P<end>\d+))?
456 (?P<end>\d+))?
457 $""", re.VERBOSE)
457 $""", re.VERBOSE)
458
458
459 def extract_hist_ranges(ranges_str):
459 def extract_hist_ranges(ranges_str):
460 """Turn a string of history ranges into 3-tuples of (session, start, stop).
460 """Turn a string of history ranges into 3-tuples of (session, start, stop).
461
461
462 Examples
462 Examples
463 --------
463 --------
464 list(extract_input_ranges("~8/5-~7/4 2"))
464 list(extract_input_ranges("~8/5-~7/4 2"))
465 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
465 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
466 """
466 """
467 for range_str in ranges_str.split():
467 for range_str in ranges_str.split():
468 rmatch = range_re.match(range_str)
468 rmatch = range_re.match(range_str)
469 if not rmatch:
469 if not rmatch:
470 continue
470 continue
471 start = int(rmatch.group("start"))
471 start = int(rmatch.group("start"))
472 end = rmatch.group("end")
472 end = rmatch.group("end")
473 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
473 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
474 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
474 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
475 end += 1
475 end += 1
476 startsess = rmatch.group("startsess") or "0"
476 startsess = rmatch.group("startsess") or "0"
477 endsess = rmatch.group("endsess") or startsess
477 endsess = rmatch.group("endsess") or startsess
478 startsess = int(startsess.replace("~","-"))
478 startsess = int(startsess.replace("~","-"))
479 endsess = int(endsess.replace("~","-"))
479 endsess = int(endsess.replace("~","-"))
480 assert endsess >= startsess
480 assert endsess >= startsess
481
481
482 if endsess == startsess:
482 if endsess == startsess:
483 yield (startsess, start, end)
483 yield (startsess, start, end)
484 continue
484 continue
485 # Multiple sessions in one range:
485 # Multiple sessions in one range:
486 yield (startsess, start, None)
486 yield (startsess, start, None)
487 for sess in range(startsess+1, endsess):
487 for sess in range(startsess+1, endsess):
488 yield (sess, 1, None)
488 yield (sess, 1, None)
489 yield (endsess, 1, end)
489 yield (endsess, 1, end)
490
490
491 def _format_lineno(session, line):
491 def _format_lineno(session, line):
492 """Helper function to format line numbers properly."""
492 """Helper function to format line numbers properly."""
493 if session == 0:
493 if session == 0:
494 return str(line)
494 return str(line)
495 return "%s#%s" % (session, line)
495 return "%s#%s" % (session, line)
496
496
497 @testdec.skip_doctest
497 @testdec.skip_doctest
498 def magic_history(self, parameter_s = ''):
498 def magic_history(self, parameter_s = ''):
499 """Print input history (_i<n> variables), with most recent last.
499 """Print input history (_i<n> variables), with most recent last.
500
500
501 %history -> print at most 40 inputs (some may be multi-line)\\
501 %history -> print at most 40 inputs (some may be multi-line)\\
502 %history n -> print at most n inputs\\
502 %history n -> print at most n inputs\\
503 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
503 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
504
504
505 By default, input history is printed without line numbers so it can be
505 By default, input history is printed without line numbers so it can be
506 directly pasted into an editor. Use -n to show them.
506 directly pasted into an editor. Use -n to show them.
507
507
508 Ranges of history can be indicated using the syntax:
508 Ranges of history can be indicated using the syntax:
509 4 : Line 4, current session
509 4 : Line 4, current session
510 4-6 : Lines 4-6, current session
510 4-6 : Lines 4-6, current session
511 243/1-5: Lines 1-5, session 243
511 243/1-5: Lines 1-5, session 243
512 ~2/7 : Line 7, session 2 before current
512 ~2/7 : Line 7, session 2 before current
513 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
513 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
514 of 6 sessions ago.
514 of 6 sessions ago.
515 Multiple ranges can be entered, separated by spaces
515 Multiple ranges can be entered, separated by spaces
516
516
517 The same syntax is used by %macro, %save, %edit, %rerun
517 The same syntax is used by %macro, %save, %edit, %rerun
518
518
519 Options:
519 Options:
520
520
521 -n: print line numbers for each input.
521 -n: print line numbers for each input.
522 This feature is only available if numbered prompts are in use.
522 This feature is only available if numbered prompts are in use.
523
523
524 -o: also print outputs for each input.
524 -o: also print outputs for each input.
525
525
526 -p: print classic '>>>' python prompts before each input. This is useful
526 -p: print classic '>>>' python prompts before each input. This is useful
527 for making documentation, and in conjunction with -o, for producing
527 for making documentation, and in conjunction with -o, for producing
528 doctest-ready output.
528 doctest-ready output.
529
529
530 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
530 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
531
531
532 -t: print the 'translated' history, as IPython understands it. IPython
532 -t: print the 'translated' history, as IPython understands it. IPython
533 filters your input and converts it all into valid Python source before
533 filters your input and converts it all into valid Python source before
534 executing it (things like magics or aliases are turned into function
534 executing it (things like magics or aliases are turned into function
535 calls, for example). With this option, you'll see the native history
535 calls, for example). With this option, you'll see the native history
536 instead of the user-entered version: '%cd /' will be seen as
536 instead of the user-entered version: '%cd /' will be seen as
537 'get_ipython().magic("%cd /")' instead of '%cd /'.
537 'get_ipython().magic("%cd /")' instead of '%cd /'.
538
538
539 -g: treat the arg as a pattern to grep for in (full) history.
539 -g: treat the arg as a pattern to grep for in (full) history.
540 This includes the saved history (almost all commands ever written).
540 This includes the saved history (almost all commands ever written).
541 Use '%hist -g' to show full saved history (may be very long).
541 Use '%hist -g' to show full saved history (may be very long).
542
542
543 -l: get the last n lines from all sessions. Specify n as a single arg, or
543 -l: get the last n lines from all sessions. Specify n as a single arg, or
544 the default is the last 10 lines.
544 the default is the last 10 lines.
545
545
546 -f FILENAME: instead of printing the output to the screen, redirect it to
546 -f FILENAME: instead of printing the output to the screen, redirect it to
547 the given file. The file is always overwritten, though IPython asks for
547 the given file. The file is always overwritten, though IPython asks for
548 confirmation first if it already exists.
548 confirmation first if it already exists.
549
549
550 Examples
550 Examples
551 --------
551 --------
552 ::
552 ::
553
553
554 In [6]: %hist -n 4 6
554 In [6]: %hist -n 4 6
555 4:a = 12
555 4:a = 12
556 5:print a**2
556 5:print a**2
557
557
558 """
558 """
559
559
560 if not self.shell.displayhook.do_full_cache:
560 if not self.shell.displayhook.do_full_cache:
561 print('This feature is only available if numbered prompts are in use.')
561 print('This feature is only available if numbered prompts are in use.')
562 return
562 return
563 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
563 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
564
564
565 # For brevity
565 # For brevity
566 history_manager = self.shell.history_manager
566 history_manager = self.shell.history_manager
567
567
568 def _format_lineno(session, line):
568 def _format_lineno(session, line):
569 """Helper function to format line numbers properly."""
569 """Helper function to format line numbers properly."""
570 if session in (0, history_manager.session_number):
570 if session in (0, history_manager.session_number):
571 return str(line)
571 return str(line)
572 return "%s/%s" % (session, line)
572 return "%s/%s" % (session, line)
573
573
574 # Check if output to specific file was requested.
574 # Check if output to specific file was requested.
575 try:
575 try:
576 outfname = opts['f']
576 outfname = opts['f']
577 except KeyError:
577 except KeyError:
578 outfile = IPython.utils.io.Term.cout # default
578 outfile = IPython.utils.io.Term.cout # default
579 # We don't want to close stdout at the end!
579 # We don't want to close stdout at the end!
580 close_at_end = False
580 close_at_end = False
581 else:
581 else:
582 if os.path.exists(outfname):
582 if os.path.exists(outfname):
583 if not ask_yes_no("File %r exists. Overwrite?" % outfname):
583 if not ask_yes_no("File %r exists. Overwrite?" % outfname):
584 print('Aborting.')
584 print('Aborting.')
585 return
585 return
586
586
587 outfile = open(outfname,'w')
587 outfile = open(outfname,'w')
588 close_at_end = True
588 close_at_end = True
589
589
590 print_nums = 'n' in opts
590 print_nums = 'n' in opts
591 get_output = 'o' in opts
591 get_output = 'o' in opts
592 pyprompts = 'p' in opts
592 pyprompts = 'p' in opts
593 # Raw history is the default
593 # Raw history is the default
594 raw = not('t' in opts)
594 raw = not('t' in opts)
595
595
596 default_length = 40
596 default_length = 40
597 pattern = None
597 pattern = None
598
598
599 if 'g' in opts: # Glob search
599 if 'g' in opts: # Glob search
600 pattern = "*" + args + "*" if args else "*"
600 pattern = "*" + args + "*" if args else "*"
601 hist = history_manager.search(pattern, raw=raw, output=get_output)
601 hist = history_manager.search(pattern, raw=raw, output=get_output)
602 elif 'l' in opts: # Get 'tail'
602 elif 'l' in opts: # Get 'tail'
603 try:
603 try:
604 n = int(args)
604 n = int(args)
605 except ValueError, IndexError:
605 except ValueError, IndexError:
606 n = 10
606 n = 10
607 hist = history_manager.get_tail(n, raw=raw, output=get_output)
607 hist = history_manager.get_tail(n, raw=raw, output=get_output)
608 else:
608 else:
609 if args: # Get history by ranges
609 if args: # Get history by ranges
610 hist = history_manager.get_range_by_str(args, raw, get_output)
610 hist = history_manager.get_range_by_str(args, raw, get_output)
611 else: # Just get history for the current session
611 else: # Just get history for the current session
612 hist = history_manager.get_range(raw=raw, output=get_output)
612 hist = history_manager.get_range(raw=raw, output=get_output)
613
613
614 # We could be displaying the entire history, so let's not try to pull it
614 # We could be displaying the entire history, so let's not try to pull it
615 # into a list in memory. Anything that needs more space will just misalign.
615 # into a list in memory. Anything that needs more space will just misalign.
616 width = 4
616 width = 4
617
617
618 for session, lineno, inline in hist:
618 for session, lineno, inline in hist:
619 # Print user history with tabs expanded to 4 spaces. The GUI clients
619 # Print user history with tabs expanded to 4 spaces. The GUI clients
620 # use hard tabs for easier usability in auto-indented code, but we want
620 # use hard tabs for easier usability in auto-indented code, but we want
621 # to produce PEP-8 compliant history for safe pasting into an editor.
621 # to produce PEP-8 compliant history for safe pasting into an editor.
622 if get_output:
622 if get_output:
623 inline, output = inline
623 inline, output = inline
624 inline = inline.expandtabs(4).rstrip()
624 inline = inline.expandtabs(4).rstrip()
625
625
626 multiline = "\n" in inline
626 multiline = "\n" in inline
627 line_sep = '\n' if multiline else ' '
627 line_sep = '\n' if multiline else ' '
628 if print_nums:
628 if print_nums:
629 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
629 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
630 line_sep), file=outfile, end='')
630 line_sep), file=outfile, end='')
631 if pyprompts:
631 if pyprompts:
632 print(">>> ", end="", file=outfile)
632 print(">>> ", end="", file=outfile)
633 if multiline:
633 if multiline:
634 inline = "\n... ".join(inline.splitlines()) + "\n..."
634 inline = "\n... ".join(inline.splitlines()) + "\n..."
635 print(inline, file=outfile)
635 print(inline, file=outfile)
636 if get_output and output:
636 if get_output and output:
637 print("\n".join(output), file=outfile)
637 print("\n".join(output), file=outfile)
638
638
639 if close_at_end:
639 if close_at_end:
640 outfile.close()
640 outfile.close()
641
641
642
642
643 def magic_rep(self, arg):
643 def magic_rep(self, arg):
644 r""" Repeat a command, or get command to input line for editing
644 r""" Repeat a command, or get command to input line for editing
645
645
646 - %rep (no arguments):
646 - %rep (no arguments):
647
647
648 Place a string version of last computation result (stored in the special '_'
648 Place a string version of last computation result (stored in the special '_'
649 variable) to the next input prompt. Allows you to create elaborate command
649 variable) to the next input prompt. Allows you to create elaborate command
650 lines without using copy-paste::
650 lines without using copy-paste::
651
651
652 In[1]: l = ["hei", "vaan"]
652 In[1]: l = ["hei", "vaan"]
653 In[2]: "".join(l)
653 In[2]: "".join(l)
654 Out[2]: heivaan
654 Out[2]: heivaan
655 In[3]: %rep
655 In[3]: %rep
656 In[4]: heivaan_ <== cursor blinking
656 In[4]: heivaan_ <== cursor blinking
657
657
658 %rep 45
658 %rep 45
659
659
660 Place history line 45 on the next input prompt. Use %hist to find
660 Place history line 45 on the next input prompt. Use %hist to find
661 out the number.
661 out the number.
662
662
663 %rep 1-4
663 %rep 1-4
664
664
665 Combine the specified lines into one cell, and place it on the next
665 Combine the specified lines into one cell, and place it on the next
666 input prompt. See %history for the slice syntax.
666 input prompt. See %history for the slice syntax.
667
667
668 %rep foo+bar
668 %rep foo+bar
669
669
670 If foo+bar can be evaluated in the user namespace, the result is
670 If foo+bar can be evaluated in the user namespace, the result is
671 placed at the next input prompt. Otherwise, the history is searched
671 placed at the next input prompt. Otherwise, the history is searched
672 for lines which contain that substring, and the most recent one is
672 for lines which contain that substring, and the most recent one is
673 placed at the next input prompt.
673 placed at the next input prompt.
674 """
674 """
675 if not arg: # Last output
675 if not arg: # Last output
676 self.set_next_input(str(self.shell.user_ns["_"]))
676 self.set_next_input(str(self.shell.user_ns["_"]))
677 return
677 return
678 # Get history range
678 # Get history range
679 histlines = self.history_manager.get_range_by_str(arg)
679 histlines = self.history_manager.get_range_by_str(arg)
680 cmd = "\n".join(x[2] for x in histlines)
680 cmd = "\n".join(x[2] for x in histlines)
681 if cmd:
681 if cmd:
682 self.set_next_input(cmd.rstrip())
682 self.set_next_input(cmd.rstrip())
683 return
683 return
684
684
685 try: # Variable in user namespace
685 try: # Variable in user namespace
686 cmd = str(eval(arg, self.shell.user_ns))
686 cmd = str(eval(arg, self.shell.user_ns))
687 except Exception: # Search for term in history
687 except Exception: # Search for term in history
688 histlines = self.history_manager.search("*"+arg+"*")
688 histlines = self.history_manager.search("*"+arg+"*")
689 for h in reversed([x[2] for x in histlines]):
689 for h in reversed([x[2] for x in histlines]):
690 if 'rep' in h:
690 if 'rep' in h:
691 continue
691 continue
692 self.set_next_input(h.rstrip())
692 self.set_next_input(h.rstrip())
693 return
693 return
694 else:
694 else:
695 self.set_next_input(cmd.rstrip())
695 self.set_next_input(cmd.rstrip())
696 print("Couldn't evaluate or find in history:", arg)
696 print("Couldn't evaluate or find in history:", arg)
697
697
698 def magic_rerun(self, parameter_s=''):
698 def magic_rerun(self, parameter_s=''):
699 """Re-run previous input
699 """Re-run previous input
700
700
701 By default, you can specify ranges of input history to be repeated
701 By default, you can specify ranges of input history to be repeated
702 (as with %history). With no arguments, it will repeat the last line.
702 (as with %history). With no arguments, it will repeat the last line.
703
703
704 Options:
704 Options:
705
705
706 -l <n> : Repeat the last n lines of input, not including the
706 -l <n> : Repeat the last n lines of input, not including the
707 current command.
707 current command.
708
708
709 -g foo : Repeat the most recent line which contains foo
709 -g foo : Repeat the most recent line which contains foo
710 """
710 """
711 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
711 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
712 if "l" in opts: # Last n lines
712 if "l" in opts: # Last n lines
713 n = int(opts['l'])
713 n = int(opts['l'])
714 hist = self.history_manager.get_tail(n)
714 hist = self.history_manager.get_tail(n)
715 elif "g" in opts: # Search
715 elif "g" in opts: # Search
716 p = "*"+opts['g']+"*"
716 p = "*"+opts['g']+"*"
717 hist = list(self.history_manager.search(p))
717 hist = list(self.history_manager.search(p))
718 for l in reversed(hist):
718 for l in reversed(hist):
719 if "rerun" not in l[2]:
719 if "rerun" not in l[2]:
720 hist = [l] # The last match which isn't a %rerun
720 hist = [l] # The last match which isn't a %rerun
721 break
721 break
722 else:
722 else:
723 hist = [] # No matches except %rerun
723 hist = [] # No matches except %rerun
724 elif args: # Specify history ranges
724 elif args: # Specify history ranges
725 hist = self.history_manager.get_range_by_str(args)
725 hist = self.history_manager.get_range_by_str(args)
726 else: # Last line
726 else: # Last line
727 hist = self.history_manager.get_tail(1)
727 hist = self.history_manager.get_tail(1)
728 hist = [x[2] for x in hist]
728 hist = [x[2] for x in hist]
729 if not hist:
729 if not hist:
730 print("No lines in history match specification")
730 print("No lines in history match specification")
731 return
731 return
732 histlines = "\n".join(hist)
732 histlines = "\n".join(hist)
733 print("=== Executing: ===")
733 print("=== Executing: ===")
734 print(histlines)
734 print(histlines)
735 print("=== Output: ===")
735 print("=== Output: ===")
736 self.run_cell("\n".join(hist), store_history=False)
736 self.run_cell("\n".join(hist), store_history=False)
737
737
738
738
739 def init_ipython(ip):
739 def init_ipython(ip):
740 ip.define_magic("rep", magic_rep)
740 ip.define_magic("rep", magic_rep)
741 ip.define_magic("recall", magic_rep)
741 ip.define_magic("recall", magic_rep)
742 ip.define_magic("rerun", magic_rerun)
742 ip.define_magic("rerun", magic_rerun)
743 ip.define_magic("hist",magic_history) # Alternative name
743 ip.define_magic("hist",magic_history) # Alternative name
744 ip.define_magic("history",magic_history)
744 ip.define_magic("history",magic_history)
745
745
746 # XXX - ipy_completers are in quarantine, need to be updated to new apis
746 # XXX - ipy_completers are in quarantine, need to be updated to new apis
747 #import ipy_completers
747 #import ipy_completers
748 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
748 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
General Comments 0
You need to be logged in to leave comments. Login now