##// END OF EJS Templates
Allow IPython to run without sqlite3...
MinRK -
Show More
@@ -1,909 +1,948 b''
1 """ History related magics and functionality """
1 """ History related magics and functionality """
2 #-----------------------------------------------------------------------------
2 #-----------------------------------------------------------------------------
3 # Copyright (C) 2010 The IPython Development Team.
3 # Copyright (C) 2010 The IPython Development Team.
4 #
4 #
5 # Distributed under the terms of the BSD License.
5 # Distributed under the terms of the BSD License.
6 #
6 #
7 # The full license is in the file COPYING.txt, distributed with this software.
7 # The full license is in the file COPYING.txt, distributed with this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 from __future__ import print_function
13 from __future__ import print_function
14
14
15 # Stdlib imports
15 # Stdlib imports
16 import atexit
16 import atexit
17 import datetime
17 import datetime
18 import os
18 import os
19 import re
19 import re
20 try:
20 import sqlite3
21 import sqlite3
22 except ImportError:
23 sqlite3 = None
21 import threading
24 import threading
22
25
23 # Our own packages
26 # Our own packages
24 from IPython.config.configurable import Configurable
27 from IPython.config.configurable import Configurable
25
28 from IPython.external.decorator import decorator
26 from IPython.testing.skipdoctest import skip_doctest
29 from IPython.testing.skipdoctest import skip_doctest
27 from IPython.utils import io
30 from IPython.utils import io
28 from IPython.utils.path import locate_profile
31 from IPython.utils.path import locate_profile
29 from IPython.utils.traitlets import Bool, Dict, Instance, Int, CInt, List, Unicode
32 from IPython.utils.traitlets import Bool, Dict, Instance, Int, CInt, List, Unicode
30 from IPython.utils.warn import warn
33 from IPython.utils.warn import warn
31
34
32 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
33 # Classes and functions
36 # Classes and functions
34 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
35
38
39 class DummyDB(object):
40 """Dummy DB that will act as a black hole for history.
41
42 Only used in the absence of sqlite"""
43 def execute(*args, **kwargs):
44 return []
45
46 def commit(self, *args, **kwargs):
47 pass
48
49 def __enter__(self, *args, **kwargs):
50 pass
51
52 def __exit__(self, *args, **kwargs):
53 pass
54
55 @decorator
56 def needs_sqlite(f,*a,**kw):
57 """return an empty list in the absence of sqlite"""
58 if sqlite3 is None:
59 return []
60 else:
61 return f(*a,**kw)
62
36 class HistoryAccessor(Configurable):
63 class HistoryAccessor(Configurable):
37 """Access the history database without adding to it.
64 """Access the history database without adding to it.
38
65
39 This is intended for use by standalone history tools. IPython shells use
66 This is intended for use by standalone history tools. IPython shells use
40 HistoryManager, below, which is a subclass of this."""
67 HistoryManager, below, which is a subclass of this."""
41 # String holding the path to the history file
68 # String holding the path to the history file
42 hist_file = Unicode(config=True)
69 hist_file = Unicode(config=True)
43
70
44 # The SQLite database
71 # The SQLite database
72 if sqlite3:
45 db = Instance(sqlite3.Connection)
73 db = Instance(sqlite3.Connection)
74 else:
75 db = Instance(DummyDB)
46
76
47 def __init__(self, profile='default', hist_file=u'', shell=None, config=None, **traits):
77 def __init__(self, profile='default', hist_file=u'', shell=None, config=None, **traits):
48 """Create a new history accessor.
78 """Create a new history accessor.
49
79
50 Parameters
80 Parameters
51 ----------
81 ----------
52 profile : str
82 profile : str
53 The name of the profile from which to open history.
83 The name of the profile from which to open history.
54 hist_file : str
84 hist_file : str
55 Path to an SQLite history database stored by IPython. If specified,
85 Path to an SQLite history database stored by IPython. If specified,
56 hist_file overrides profile.
86 hist_file overrides profile.
57 shell :
87 shell :
58 InteractiveShell object, for use by HistoryManager subclass
88 InteractiveShell object, for use by HistoryManager subclass
59 config :
89 config :
60 Config object. hist_file can also be set through this.
90 Config object. hist_file can also be set through this.
61 """
91 """
62 # We need a pointer back to the shell for various tasks.
92 # We need a pointer back to the shell for various tasks.
63 super(HistoryAccessor, self).__init__(shell=shell, config=config,
93 super(HistoryAccessor, self).__init__(shell=shell, config=config,
64 hist_file=hist_file, **traits)
94 hist_file=hist_file, **traits)
65
95
66 if self.hist_file == u'':
96 if self.hist_file == u'':
67 # No one has set the hist_file, yet.
97 # No one has set the hist_file, yet.
68 self.hist_file = self._get_hist_file_name(profile)
98 self.hist_file = self._get_hist_file_name(profile)
69
99
100 if sqlite3 is None:
101 warn("IPython History requires SQLite, your history will not be saved\n")
102 self.db = DummyDB()
103 return
104
70 try:
105 try:
71 self.init_db()
106 self.init_db()
72 except sqlite3.DatabaseError:
107 except sqlite3.DatabaseError:
73 if os.path.isfile(self.hist_file):
108 if os.path.isfile(self.hist_file):
74 # Try to move the file out of the way.
109 # Try to move the file out of the way.
75 newpath = os.path.join(self.shell.profile_dir.location, "hist-corrupt.sqlite")
110 newpath = os.path.join(self.shell.profile_dir.location, "hist-corrupt.sqlite")
76 os.rename(self.hist_file, newpath)
111 os.rename(self.hist_file, newpath)
77 print("ERROR! History file wasn't a valid SQLite database.",
112 print("ERROR! History file wasn't a valid SQLite database.",
78 "It was moved to %s" % newpath, "and a new file created.")
113 "It was moved to %s" % newpath, "and a new file created.")
79 self.init_db()
114 self.init_db()
80 else:
115 else:
81 # The hist_file is probably :memory: or something else.
116 # The hist_file is probably :memory: or something else.
82 raise
117 raise
83
118
84 def _get_hist_file_name(self, profile='default'):
119 def _get_hist_file_name(self, profile='default'):
85 """Find the history file for the given profile name.
120 """Find the history file for the given profile name.
86
121
87 This is overridden by the HistoryManager subclass, to use the shell's
122 This is overridden by the HistoryManager subclass, to use the shell's
88 active profile.
123 active profile.
89
124
90 Parameters
125 Parameters
91 ----------
126 ----------
92 profile : str
127 profile : str
93 The name of a profile which has a history file.
128 The name of a profile which has a history file.
94 """
129 """
95 return os.path.join(locate_profile(profile), 'history.sqlite')
130 return os.path.join(locate_profile(profile), 'history.sqlite')
96
131
97 def init_db(self):
132 def init_db(self):
98 """Connect to the database, and create tables if necessary."""
133 """Connect to the database, and create tables if necessary."""
99 # use detect_types so that timestamps return datetime objects
134 # use detect_types so that timestamps return datetime objects
100 self.db = sqlite3.connect(self.hist_file, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
135 self.db = sqlite3.connect(self.hist_file, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
101 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
136 self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
102 primary key autoincrement, start timestamp,
137 primary key autoincrement, start timestamp,
103 end timestamp, num_cmds integer, remark text)""")
138 end timestamp, num_cmds integer, remark text)""")
104 self.db.execute("""CREATE TABLE IF NOT EXISTS history
139 self.db.execute("""CREATE TABLE IF NOT EXISTS history
105 (session integer, line integer, source text, source_raw text,
140 (session integer, line integer, source text, source_raw text,
106 PRIMARY KEY (session, line))""")
141 PRIMARY KEY (session, line))""")
107 # Output history is optional, but ensure the table's there so it can be
142 # Output history is optional, but ensure the table's there so it can be
108 # enabled later.
143 # enabled later.
109 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
144 self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
110 (session integer, line integer, output text,
145 (session integer, line integer, output text,
111 PRIMARY KEY (session, line))""")
146 PRIMARY KEY (session, line))""")
112 self.db.commit()
147 self.db.commit()
113
148
114 def writeout_cache(self):
149 def writeout_cache(self):
115 """Overridden by HistoryManager to dump the cache before certain
150 """Overridden by HistoryManager to dump the cache before certain
116 database lookups."""
151 database lookups."""
117 pass
152 pass
118
153
119 ## -------------------------------
154 ## -------------------------------
120 ## Methods for retrieving history:
155 ## Methods for retrieving history:
121 ## -------------------------------
156 ## -------------------------------
122 def _run_sql(self, sql, params, raw=True, output=False):
157 def _run_sql(self, sql, params, raw=True, output=False):
123 """Prepares and runs an SQL query for the history database.
158 """Prepares and runs an SQL query for the history database.
124
159
125 Parameters
160 Parameters
126 ----------
161 ----------
127 sql : str
162 sql : str
128 Any filtering expressions to go after SELECT ... FROM ...
163 Any filtering expressions to go after SELECT ... FROM ...
129 params : tuple
164 params : tuple
130 Parameters passed to the SQL query (to replace "?")
165 Parameters passed to the SQL query (to replace "?")
131 raw, output : bool
166 raw, output : bool
132 See :meth:`get_range`
167 See :meth:`get_range`
133
168
134 Returns
169 Returns
135 -------
170 -------
136 Tuples as :meth:`get_range`
171 Tuples as :meth:`get_range`
137 """
172 """
138 toget = 'source_raw' if raw else 'source'
173 toget = 'source_raw' if raw else 'source'
139 sqlfrom = "history"
174 sqlfrom = "history"
140 if output:
175 if output:
141 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
176 sqlfrom = "history LEFT JOIN output_history USING (session, line)"
142 toget = "history.%s, output_history.output" % toget
177 toget = "history.%s, output_history.output" % toget
143 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
178 cur = self.db.execute("SELECT session, line, %s FROM %s " %\
144 (toget, sqlfrom) + sql, params)
179 (toget, sqlfrom) + sql, params)
145 if output: # Regroup into 3-tuples, and parse JSON
180 if output: # Regroup into 3-tuples, and parse JSON
146 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
181 return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
147 return cur
182 return cur
148
183
184 @needs_sqlite
149 def get_session_info(self, session=0):
185 def get_session_info(self, session=0):
150 """get info about a session
186 """get info about a session
151
187
152 Parameters
188 Parameters
153 ----------
189 ----------
154
190
155 session : int
191 session : int
156 Session number to retrieve. The current session is 0, and negative
192 Session number to retrieve. The current session is 0, and negative
157 numbers count back from current session, so -1 is previous session.
193 numbers count back from current session, so -1 is previous session.
158
194
159 Returns
195 Returns
160 -------
196 -------
161
197
162 (session_id [int], start [datetime], end [datetime], num_cmds [int], remark [unicode])
198 (session_id [int], start [datetime], end [datetime], num_cmds [int], remark [unicode])
163
199
164 Sessions that are running or did not exit cleanly will have `end=None`
200 Sessions that are running or did not exit cleanly will have `end=None`
165 and `num_cmds=None`.
201 and `num_cmds=None`.
166
202
167 """
203 """
168
204
169 if session <= 0:
205 if session <= 0:
170 session += self.session_number
206 session += self.session_number
171
207
172 query = "SELECT * from sessions where session == ?"
208 query = "SELECT * from sessions where session == ?"
173 return self.db.execute(query, (session,)).fetchone()
209 return self.db.execute(query, (session,)).fetchone()
174
210
175 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
211 def get_tail(self, n=10, raw=True, output=False, include_latest=False):
176 """Get the last n lines from the history database.
212 """Get the last n lines from the history database.
177
213
178 Parameters
214 Parameters
179 ----------
215 ----------
180 n : int
216 n : int
181 The number of lines to get
217 The number of lines to get
182 raw, output : bool
218 raw, output : bool
183 See :meth:`get_range`
219 See :meth:`get_range`
184 include_latest : bool
220 include_latest : bool
185 If False (default), n+1 lines are fetched, and the latest one
221 If False (default), n+1 lines are fetched, and the latest one
186 is discarded. This is intended to be used where the function
222 is discarded. This is intended to be used where the function
187 is called by a user command, which it should not return.
223 is called by a user command, which it should not return.
188
224
189 Returns
225 Returns
190 -------
226 -------
191 Tuples as :meth:`get_range`
227 Tuples as :meth:`get_range`
192 """
228 """
193 self.writeout_cache()
229 self.writeout_cache()
194 if not include_latest:
230 if not include_latest:
195 n += 1
231 n += 1
196 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
232 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
197 (n,), raw=raw, output=output)
233 (n,), raw=raw, output=output)
198 if not include_latest:
234 if not include_latest:
199 return reversed(list(cur)[1:])
235 return reversed(list(cur)[1:])
200 return reversed(list(cur))
236 return reversed(list(cur))
201
237
202 def search(self, pattern="*", raw=True, search_raw=True,
238 def search(self, pattern="*", raw=True, search_raw=True,
203 output=False):
239 output=False):
204 """Search the database using unix glob-style matching (wildcards
240 """Search the database using unix glob-style matching (wildcards
205 * and ?).
241 * and ?).
206
242
207 Parameters
243 Parameters
208 ----------
244 ----------
209 pattern : str
245 pattern : str
210 The wildcarded pattern to match when searching
246 The wildcarded pattern to match when searching
211 search_raw : bool
247 search_raw : bool
212 If True, search the raw input, otherwise, the parsed input
248 If True, search the raw input, otherwise, the parsed input
213 raw, output : bool
249 raw, output : bool
214 See :meth:`get_range`
250 See :meth:`get_range`
215
251
216 Returns
252 Returns
217 -------
253 -------
218 Tuples as :meth:`get_range`
254 Tuples as :meth:`get_range`
219 """
255 """
220 tosearch = "source_raw" if search_raw else "source"
256 tosearch = "source_raw" if search_raw else "source"
221 if output:
257 if output:
222 tosearch = "history." + tosearch
258 tosearch = "history." + tosearch
223 self.writeout_cache()
259 self.writeout_cache()
224 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
260 return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
225 raw=raw, output=output)
261 raw=raw, output=output)
226
262
227 def get_range(self, session, start=1, stop=None, raw=True,output=False):
263 def get_range(self, session, start=1, stop=None, raw=True,output=False):
228 """Retrieve input by session.
264 """Retrieve input by session.
229
265
230 Parameters
266 Parameters
231 ----------
267 ----------
232 session : int
268 session : int
233 Session number to retrieve.
269 Session number to retrieve.
234 start : int
270 start : int
235 First line to retrieve.
271 First line to retrieve.
236 stop : int
272 stop : int
237 End of line range (excluded from output itself). If None, retrieve
273 End of line range (excluded from output itself). If None, retrieve
238 to the end of the session.
274 to the end of the session.
239 raw : bool
275 raw : bool
240 If True, return untranslated input
276 If True, return untranslated input
241 output : bool
277 output : bool
242 If True, attempt to include output. This will be 'real' Python
278 If True, attempt to include output. This will be 'real' Python
243 objects for the current session, or text reprs from previous
279 objects for the current session, or text reprs from previous
244 sessions if db_log_output was enabled at the time. Where no output
280 sessions if db_log_output was enabled at the time. Where no output
245 is found, None is used.
281 is found, None is used.
246
282
247 Returns
283 Returns
248 -------
284 -------
249 An iterator over the desired lines. Each line is a 3-tuple, either
285 An iterator over the desired lines. Each line is a 3-tuple, either
250 (session, line, input) if output is False, or
286 (session, line, input) if output is False, or
251 (session, line, (input, output)) if output is True.
287 (session, line, (input, output)) if output is True.
252 """
288 """
253 if stop:
289 if stop:
254 lineclause = "line >= ? AND line < ?"
290 lineclause = "line >= ? AND line < ?"
255 params = (session, start, stop)
291 params = (session, start, stop)
256 else:
292 else:
257 lineclause = "line>=?"
293 lineclause = "line>=?"
258 params = (session, start)
294 params = (session, start)
259
295
260 return self._run_sql("WHERE session==? AND %s""" % lineclause,
296 return self._run_sql("WHERE session==? AND %s""" % lineclause,
261 params, raw=raw, output=output)
297 params, raw=raw, output=output)
262
298
263 def get_range_by_str(self, rangestr, raw=True, output=False):
299 def get_range_by_str(self, rangestr, raw=True, output=False):
264 """Get lines of history from a string of ranges, as used by magic
300 """Get lines of history from a string of ranges, as used by magic
265 commands %hist, %save, %macro, etc.
301 commands %hist, %save, %macro, etc.
266
302
267 Parameters
303 Parameters
268 ----------
304 ----------
269 rangestr : str
305 rangestr : str
270 A string specifying ranges, e.g. "5 ~2/1-4". See
306 A string specifying ranges, e.g. "5 ~2/1-4". See
271 :func:`magic_history` for full details.
307 :func:`magic_history` for full details.
272 raw, output : bool
308 raw, output : bool
273 As :meth:`get_range`
309 As :meth:`get_range`
274
310
275 Returns
311 Returns
276 -------
312 -------
277 Tuples as :meth:`get_range`
313 Tuples as :meth:`get_range`
278 """
314 """
279 for sess, s, e in extract_hist_ranges(rangestr):
315 for sess, s, e in extract_hist_ranges(rangestr):
280 for line in self.get_range(sess, s, e, raw=raw, output=output):
316 for line in self.get_range(sess, s, e, raw=raw, output=output):
281 yield line
317 yield line
282
318
283
319
284 class HistoryManager(HistoryAccessor):
320 class HistoryManager(HistoryAccessor):
285 """A class to organize all history-related functionality in one place.
321 """A class to organize all history-related functionality in one place.
286 """
322 """
287 # Public interface
323 # Public interface
288
324
289 # An instance of the IPython shell we are attached to
325 # An instance of the IPython shell we are attached to
290 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
326 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
291 # Lists to hold processed and raw history. These start with a blank entry
327 # Lists to hold processed and raw history. These start with a blank entry
292 # so that we can index them starting from 1
328 # so that we can index them starting from 1
293 input_hist_parsed = List([""])
329 input_hist_parsed = List([""])
294 input_hist_raw = List([""])
330 input_hist_raw = List([""])
295 # A list of directories visited during session
331 # A list of directories visited during session
296 dir_hist = List()
332 dir_hist = List()
297 def _dir_hist_default(self):
333 def _dir_hist_default(self):
298 try:
334 try:
299 return [os.getcwdu()]
335 return [os.getcwdu()]
300 except OSError:
336 except OSError:
301 return []
337 return []
302
338
303 # A dict of output history, keyed with ints from the shell's
339 # A dict of output history, keyed with ints from the shell's
304 # execution count.
340 # execution count.
305 output_hist = Dict()
341 output_hist = Dict()
306 # The text/plain repr of outputs.
342 # The text/plain repr of outputs.
307 output_hist_reprs = Dict()
343 output_hist_reprs = Dict()
308
344
309 # The number of the current session in the history database
345 # The number of the current session in the history database
310 session_number = CInt()
346 session_number = CInt()
311 # Should we log output to the database? (default no)
347 # Should we log output to the database? (default no)
312 db_log_output = Bool(False, config=True)
348 db_log_output = Bool(False, config=True)
313 # Write to database every x commands (higher values save disk access & power)
349 # Write to database every x commands (higher values save disk access & power)
314 # Values of 1 or less effectively disable caching.
350 # Values of 1 or less effectively disable caching.
315 db_cache_size = Int(0, config=True)
351 db_cache_size = Int(0, config=True)
316 # The input and output caches
352 # The input and output caches
317 db_input_cache = List()
353 db_input_cache = List()
318 db_output_cache = List()
354 db_output_cache = List()
319
355
320 # History saving in separate thread
356 # History saving in separate thread
321 save_thread = Instance('IPython.core.history.HistorySavingThread')
357 save_thread = Instance('IPython.core.history.HistorySavingThread')
322 try: # Event is a function returning an instance of _Event...
358 try: # Event is a function returning an instance of _Event...
323 save_flag = Instance(threading._Event)
359 save_flag = Instance(threading._Event)
324 except AttributeError: # ...until Python 3.3, when it's a class.
360 except AttributeError: # ...until Python 3.3, when it's a class.
325 save_flag = Instance(threading.Event)
361 save_flag = Instance(threading.Event)
326
362
327 # Private interface
363 # Private interface
328 # Variables used to store the three last inputs from the user. On each new
364 # Variables used to store the three last inputs from the user. On each new
329 # history update, we populate the user's namespace with these, shifted as
365 # history update, we populate the user's namespace with these, shifted as
330 # necessary.
366 # necessary.
331 _i00 = Unicode(u'')
367 _i00 = Unicode(u'')
332 _i = Unicode(u'')
368 _i = Unicode(u'')
333 _ii = Unicode(u'')
369 _ii = Unicode(u'')
334 _iii = Unicode(u'')
370 _iii = Unicode(u'')
335
371
336 # A regex matching all forms of the exit command, so that we don't store
372 # A regex matching all forms of the exit command, so that we don't store
337 # them in the history (it's annoying to rewind the first entry and land on
373 # them in the history (it's annoying to rewind the first entry and land on
338 # an exit call).
374 # an exit call).
339 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
375 _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
340
376
341 def __init__(self, shell=None, config=None, **traits):
377 def __init__(self, shell=None, config=None, **traits):
342 """Create a new history manager associated with a shell instance.
378 """Create a new history manager associated with a shell instance.
343 """
379 """
344 # We need a pointer back to the shell for various tasks.
380 # We need a pointer back to the shell for various tasks.
345 super(HistoryManager, self).__init__(shell=shell, config=config,
381 super(HistoryManager, self).__init__(shell=shell, config=config,
346 **traits)
382 **traits)
347 self.save_flag = threading.Event()
383 self.save_flag = threading.Event()
348 self.db_input_cache_lock = threading.Lock()
384 self.db_input_cache_lock = threading.Lock()
349 self.db_output_cache_lock = threading.Lock()
385 self.db_output_cache_lock = threading.Lock()
350 self.save_thread = HistorySavingThread(self)
386 self.save_thread = HistorySavingThread(self)
351 self.save_thread.start()
387 self.save_thread.start()
352
388
353 self.new_session()
389 self.new_session()
354
390
355 def _get_hist_file_name(self, profile=None):
391 def _get_hist_file_name(self, profile=None):
356 """Get default history file name based on the Shell's profile.
392 """Get default history file name based on the Shell's profile.
357
393
358 The profile parameter is ignored, but must exist for compatibility with
394 The profile parameter is ignored, but must exist for compatibility with
359 the parent class."""
395 the parent class."""
360 profile_dir = self.shell.profile_dir.location
396 profile_dir = self.shell.profile_dir.location
361 return os.path.join(profile_dir, 'history.sqlite')
397 return os.path.join(profile_dir, 'history.sqlite')
362
398
399 @needs_sqlite
363 def new_session(self, conn=None):
400 def new_session(self, conn=None):
364 """Get a new session number."""
401 """Get a new session number."""
365 if conn is None:
402 if conn is None:
366 conn = self.db
403 conn = self.db
367
404
368 with conn:
405 with conn:
369 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
406 cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
370 NULL, "") """, (datetime.datetime.now(),))
407 NULL, "") """, (datetime.datetime.now(),))
371 self.session_number = cur.lastrowid
408 self.session_number = cur.lastrowid
372
409
373 def end_session(self):
410 def end_session(self):
374 """Close the database session, filling in the end time and line count."""
411 """Close the database session, filling in the end time and line count."""
375 self.writeout_cache()
412 self.writeout_cache()
376 with self.db:
413 with self.db:
377 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
414 self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
378 session==?""", (datetime.datetime.now(),
415 session==?""", (datetime.datetime.now(),
379 len(self.input_hist_parsed)-1, self.session_number))
416 len(self.input_hist_parsed)-1, self.session_number))
380 self.session_number = 0
417 self.session_number = 0
381
418
382 def name_session(self, name):
419 def name_session(self, name):
383 """Give the current session a name in the history database."""
420 """Give the current session a name in the history database."""
384 with self.db:
421 with self.db:
385 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
422 self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
386 (name, self.session_number))
423 (name, self.session_number))
387
424
388 def reset(self, new_session=True):
425 def reset(self, new_session=True):
389 """Clear the session history, releasing all object references, and
426 """Clear the session history, releasing all object references, and
390 optionally open a new session."""
427 optionally open a new session."""
391 self.output_hist.clear()
428 self.output_hist.clear()
392 # The directory history can't be completely empty
429 # The directory history can't be completely empty
393 self.dir_hist[:] = [os.getcwdu()]
430 self.dir_hist[:] = [os.getcwdu()]
394
431
395 if new_session:
432 if new_session:
396 if self.session_number:
433 if self.session_number:
397 self.end_session()
434 self.end_session()
398 self.input_hist_parsed[:] = [""]
435 self.input_hist_parsed[:] = [""]
399 self.input_hist_raw[:] = [""]
436 self.input_hist_raw[:] = [""]
400 self.new_session()
437 self.new_session()
401
438
402 # ------------------------------
439 # ------------------------------
403 # Methods for retrieving history
440 # Methods for retrieving history
404 # ------------------------------
441 # ------------------------------
405 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
442 def _get_range_session(self, start=1, stop=None, raw=True, output=False):
406 """Get input and output history from the current session. Called by
443 """Get input and output history from the current session. Called by
407 get_range, and takes similar parameters."""
444 get_range, and takes similar parameters."""
408 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
445 input_hist = self.input_hist_raw if raw else self.input_hist_parsed
409
446
410 n = len(input_hist)
447 n = len(input_hist)
411 if start < 0:
448 if start < 0:
412 start += n
449 start += n
413 if not stop or (stop > n):
450 if not stop or (stop > n):
414 stop = n
451 stop = n
415 elif stop < 0:
452 elif stop < 0:
416 stop += n
453 stop += n
417
454
418 for i in range(start, stop):
455 for i in range(start, stop):
419 if output:
456 if output:
420 line = (input_hist[i], self.output_hist_reprs.get(i))
457 line = (input_hist[i], self.output_hist_reprs.get(i))
421 else:
458 else:
422 line = input_hist[i]
459 line = input_hist[i]
423 yield (0, i, line)
460 yield (0, i, line)
424
461
425 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
462 def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
426 """Retrieve input by session.
463 """Retrieve input by session.
427
464
428 Parameters
465 Parameters
429 ----------
466 ----------
430 session : int
467 session : int
431 Session number to retrieve. The current session is 0, and negative
468 Session number to retrieve. The current session is 0, and negative
432 numbers count back from current session, so -1 is previous session.
469 numbers count back from current session, so -1 is previous session.
433 start : int
470 start : int
434 First line to retrieve.
471 First line to retrieve.
435 stop : int
472 stop : int
436 End of line range (excluded from output itself). If None, retrieve
473 End of line range (excluded from output itself). If None, retrieve
437 to the end of the session.
474 to the end of the session.
438 raw : bool
475 raw : bool
439 If True, return untranslated input
476 If True, return untranslated input
440 output : bool
477 output : bool
441 If True, attempt to include output. This will be 'real' Python
478 If True, attempt to include output. This will be 'real' Python
442 objects for the current session, or text reprs from previous
479 objects for the current session, or text reprs from previous
443 sessions if db_log_output was enabled at the time. Where no output
480 sessions if db_log_output was enabled at the time. Where no output
444 is found, None is used.
481 is found, None is used.
445
482
446 Returns
483 Returns
447 -------
484 -------
448 An iterator over the desired lines. Each line is a 3-tuple, either
485 An iterator over the desired lines. Each line is a 3-tuple, either
449 (session, line, input) if output is False, or
486 (session, line, input) if output is False, or
450 (session, line, (input, output)) if output is True.
487 (session, line, (input, output)) if output is True.
451 """
488 """
452 if session <= 0:
489 if session <= 0:
453 session += self.session_number
490 session += self.session_number
454 if session==self.session_number: # Current session
491 if session==self.session_number: # Current session
455 return self._get_range_session(start, stop, raw, output)
492 return self._get_range_session(start, stop, raw, output)
456 return super(HistoryManager, self).get_range(session, start, stop, raw, output)
493 return super(HistoryManager, self).get_range(session, start, stop, raw, output)
457
494
458 ## ----------------------------
495 ## ----------------------------
459 ## Methods for storing history:
496 ## Methods for storing history:
460 ## ----------------------------
497 ## ----------------------------
461 def store_inputs(self, line_num, source, source_raw=None):
498 def store_inputs(self, line_num, source, source_raw=None):
462 """Store source and raw input in history and create input cache
499 """Store source and raw input in history and create input cache
463 variables _i*.
500 variables _i*.
464
501
465 Parameters
502 Parameters
466 ----------
503 ----------
467 line_num : int
504 line_num : int
468 The prompt number of this input.
505 The prompt number of this input.
469
506
470 source : str
507 source : str
471 Python input.
508 Python input.
472
509
473 source_raw : str, optional
510 source_raw : str, optional
474 If given, this is the raw input without any IPython transformations
511 If given, this is the raw input without any IPython transformations
475 applied to it. If not given, ``source`` is used.
512 applied to it. If not given, ``source`` is used.
476 """
513 """
477 if source_raw is None:
514 if source_raw is None:
478 source_raw = source
515 source_raw = source
479 source = source.rstrip('\n')
516 source = source.rstrip('\n')
480 source_raw = source_raw.rstrip('\n')
517 source_raw = source_raw.rstrip('\n')
481
518
482 # do not store exit/quit commands
519 # do not store exit/quit commands
483 if self._exit_re.match(source_raw.strip()):
520 if self._exit_re.match(source_raw.strip()):
484 return
521 return
485
522
486 self.input_hist_parsed.append(source)
523 self.input_hist_parsed.append(source)
487 self.input_hist_raw.append(source_raw)
524 self.input_hist_raw.append(source_raw)
488
525
489 with self.db_input_cache_lock:
526 with self.db_input_cache_lock:
490 self.db_input_cache.append((line_num, source, source_raw))
527 self.db_input_cache.append((line_num, source, source_raw))
491 # Trigger to flush cache and write to DB.
528 # Trigger to flush cache and write to DB.
492 if len(self.db_input_cache) >= self.db_cache_size:
529 if len(self.db_input_cache) >= self.db_cache_size:
493 self.save_flag.set()
530 self.save_flag.set()
494
531
495 # update the auto _i variables
532 # update the auto _i variables
496 self._iii = self._ii
533 self._iii = self._ii
497 self._ii = self._i
534 self._ii = self._i
498 self._i = self._i00
535 self._i = self._i00
499 self._i00 = source_raw
536 self._i00 = source_raw
500
537
501 # hackish access to user namespace to create _i1,_i2... dynamically
538 # hackish access to user namespace to create _i1,_i2... dynamically
502 new_i = '_i%s' % line_num
539 new_i = '_i%s' % line_num
503 to_main = {'_i': self._i,
540 to_main = {'_i': self._i,
504 '_ii': self._ii,
541 '_ii': self._ii,
505 '_iii': self._iii,
542 '_iii': self._iii,
506 new_i : self._i00 }
543 new_i : self._i00 }
507 self.shell.user_ns.update(to_main)
544 self.shell.user_ns.update(to_main)
508
545
509 def store_output(self, line_num):
546 def store_output(self, line_num):
510 """If database output logging is enabled, this saves all the
547 """If database output logging is enabled, this saves all the
511 outputs from the indicated prompt number to the database. It's
548 outputs from the indicated prompt number to the database. It's
512 called by run_cell after code has been executed.
549 called by run_cell after code has been executed.
513
550
514 Parameters
551 Parameters
515 ----------
552 ----------
516 line_num : int
553 line_num : int
517 The line number from which to save outputs
554 The line number from which to save outputs
518 """
555 """
519 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
556 if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
520 return
557 return
521 output = self.output_hist_reprs[line_num]
558 output = self.output_hist_reprs[line_num]
522
559
523 with self.db_output_cache_lock:
560 with self.db_output_cache_lock:
524 self.db_output_cache.append((line_num, output))
561 self.db_output_cache.append((line_num, output))
525 if self.db_cache_size <= 1:
562 if self.db_cache_size <= 1:
526 self.save_flag.set()
563 self.save_flag.set()
527
564
528 def _writeout_input_cache(self, conn):
565 def _writeout_input_cache(self, conn):
529 with conn:
566 with conn:
530 for line in self.db_input_cache:
567 for line in self.db_input_cache:
531 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
568 conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
532 (self.session_number,)+line)
569 (self.session_number,)+line)
533
570
534 def _writeout_output_cache(self, conn):
571 def _writeout_output_cache(self, conn):
535 with conn:
572 with conn:
536 for line in self.db_output_cache:
573 for line in self.db_output_cache:
537 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
574 conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
538 (self.session_number,)+line)
575 (self.session_number,)+line)
539
576
577 @needs_sqlite
540 def writeout_cache(self, conn=None):
578 def writeout_cache(self, conn=None):
541 """Write any entries in the cache to the database."""
579 """Write any entries in the cache to the database."""
542 if conn is None:
580 if conn is None:
543 conn = self.db
581 conn = self.db
544
582
545 with self.db_input_cache_lock:
583 with self.db_input_cache_lock:
546 try:
584 try:
547 self._writeout_input_cache(conn)
585 self._writeout_input_cache(conn)
548 except sqlite3.IntegrityError:
586 except sqlite3.IntegrityError:
549 self.new_session(conn)
587 self.new_session(conn)
550 print("ERROR! Session/line number was not unique in",
588 print("ERROR! Session/line number was not unique in",
551 "database. History logging moved to new session",
589 "database. History logging moved to new session",
552 self.session_number)
590 self.session_number)
553 try: # Try writing to the new session. If this fails, don't recurse
591 try: # Try writing to the new session. If this fails, don't recurse
554 self._writeout_input_cache(conn)
592 self._writeout_input_cache(conn)
555 except sqlite3.IntegrityError:
593 except sqlite3.IntegrityError:
556 pass
594 pass
557 finally:
595 finally:
558 self.db_input_cache = []
596 self.db_input_cache = []
559
597
560 with self.db_output_cache_lock:
598 with self.db_output_cache_lock:
561 try:
599 try:
562 self._writeout_output_cache(conn)
600 self._writeout_output_cache(conn)
563 except sqlite3.IntegrityError:
601 except sqlite3.IntegrityError:
564 print("!! Session/line number for output was not unique",
602 print("!! Session/line number for output was not unique",
565 "in database. Output will not be stored.")
603 "in database. Output will not be stored.")
566 finally:
604 finally:
567 self.db_output_cache = []
605 self.db_output_cache = []
568
606
569
607
570 class HistorySavingThread(threading.Thread):
608 class HistorySavingThread(threading.Thread):
571 """This thread takes care of writing history to the database, so that
609 """This thread takes care of writing history to the database, so that
572 the UI isn't held up while that happens.
610 the UI isn't held up while that happens.
573
611
574 It waits for the HistoryManager's save_flag to be set, then writes out
612 It waits for the HistoryManager's save_flag to be set, then writes out
575 the history cache. The main thread is responsible for setting the flag when
613 the history cache. The main thread is responsible for setting the flag when
576 the cache size reaches a defined threshold."""
614 the cache size reaches a defined threshold."""
577 daemon = True
615 daemon = True
578 stop_now = False
616 stop_now = False
579 def __init__(self, history_manager):
617 def __init__(self, history_manager):
580 super(HistorySavingThread, self).__init__()
618 super(HistorySavingThread, self).__init__()
581 self.history_manager = history_manager
619 self.history_manager = history_manager
582 atexit.register(self.stop)
620 atexit.register(self.stop)
583
621
622 @needs_sqlite
584 def run(self):
623 def run(self):
585 # We need a separate db connection per thread:
624 # We need a separate db connection per thread:
586 try:
625 try:
587 self.db = sqlite3.connect(self.history_manager.hist_file)
626 self.db = sqlite3.connect(self.history_manager.hist_file)
588 while True:
627 while True:
589 self.history_manager.save_flag.wait()
628 self.history_manager.save_flag.wait()
590 if self.stop_now:
629 if self.stop_now:
591 return
630 return
592 self.history_manager.save_flag.clear()
631 self.history_manager.save_flag.clear()
593 self.history_manager.writeout_cache(self.db)
632 self.history_manager.writeout_cache(self.db)
594 except Exception as e:
633 except Exception as e:
595 print(("The history saving thread hit an unexpected error (%s)."
634 print(("The history saving thread hit an unexpected error (%s)."
596 "History will not be written to the database.") % repr(e))
635 "History will not be written to the database.") % repr(e))
597
636
598 def stop(self):
637 def stop(self):
599 """This can be called from the main thread to safely stop this thread.
638 """This can be called from the main thread to safely stop this thread.
600
639
601 Note that it does not attempt to write out remaining history before
640 Note that it does not attempt to write out remaining history before
602 exiting. That should be done by calling the HistoryManager's
641 exiting. That should be done by calling the HistoryManager's
603 end_session method."""
642 end_session method."""
604 self.stop_now = True
643 self.stop_now = True
605 self.history_manager.save_flag.set()
644 self.history_manager.save_flag.set()
606 self.join()
645 self.join()
607
646
608
647
609 # To match, e.g. ~5/8-~2/3
648 # To match, e.g. ~5/8-~2/3
610 range_re = re.compile(r"""
649 range_re = re.compile(r"""
611 ((?P<startsess>~?\d+)/)?
650 ((?P<startsess>~?\d+)/)?
612 (?P<start>\d+) # Only the start line num is compulsory
651 (?P<start>\d+) # Only the start line num is compulsory
613 ((?P<sep>[\-:])
652 ((?P<sep>[\-:])
614 ((?P<endsess>~?\d+)/)?
653 ((?P<endsess>~?\d+)/)?
615 (?P<end>\d+))?
654 (?P<end>\d+))?
616 $""", re.VERBOSE)
655 $""", re.VERBOSE)
617
656
618 def extract_hist_ranges(ranges_str):
657 def extract_hist_ranges(ranges_str):
619 """Turn a string of history ranges into 3-tuples of (session, start, stop).
658 """Turn a string of history ranges into 3-tuples of (session, start, stop).
620
659
621 Examples
660 Examples
622 --------
661 --------
623 list(extract_input_ranges("~8/5-~7/4 2"))
662 list(extract_input_ranges("~8/5-~7/4 2"))
624 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
663 [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
625 """
664 """
626 for range_str in ranges_str.split():
665 for range_str in ranges_str.split():
627 rmatch = range_re.match(range_str)
666 rmatch = range_re.match(range_str)
628 if not rmatch:
667 if not rmatch:
629 continue
668 continue
630 start = int(rmatch.group("start"))
669 start = int(rmatch.group("start"))
631 end = rmatch.group("end")
670 end = rmatch.group("end")
632 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
671 end = int(end) if end else start+1 # If no end specified, get (a, a+1)
633 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
672 if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
634 end += 1
673 end += 1
635 startsess = rmatch.group("startsess") or "0"
674 startsess = rmatch.group("startsess") or "0"
636 endsess = rmatch.group("endsess") or startsess
675 endsess = rmatch.group("endsess") or startsess
637 startsess = int(startsess.replace("~","-"))
676 startsess = int(startsess.replace("~","-"))
638 endsess = int(endsess.replace("~","-"))
677 endsess = int(endsess.replace("~","-"))
639 assert endsess >= startsess
678 assert endsess >= startsess
640
679
641 if endsess == startsess:
680 if endsess == startsess:
642 yield (startsess, start, end)
681 yield (startsess, start, end)
643 continue
682 continue
644 # Multiple sessions in one range:
683 # Multiple sessions in one range:
645 yield (startsess, start, None)
684 yield (startsess, start, None)
646 for sess in range(startsess+1, endsess):
685 for sess in range(startsess+1, endsess):
647 yield (sess, 1, None)
686 yield (sess, 1, None)
648 yield (endsess, 1, end)
687 yield (endsess, 1, end)
649
688
650 def _format_lineno(session, line):
689 def _format_lineno(session, line):
651 """Helper function to format line numbers properly."""
690 """Helper function to format line numbers properly."""
652 if session == 0:
691 if session == 0:
653 return str(line)
692 return str(line)
654 return "%s#%s" % (session, line)
693 return "%s#%s" % (session, line)
655
694
656 @skip_doctest
695 @skip_doctest
657 def magic_history(self, parameter_s = ''):
696 def magic_history(self, parameter_s = ''):
658 """Print input history (_i<n> variables), with most recent last.
697 """Print input history (_i<n> variables), with most recent last.
659
698
660 %history -> print at most 40 inputs (some may be multi-line)\\
699 %history -> print at most 40 inputs (some may be multi-line)\\
661 %history n -> print at most n inputs\\
700 %history n -> print at most n inputs\\
662 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
701 %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
663
702
664 By default, input history is printed without line numbers so it can be
703 By default, input history is printed without line numbers so it can be
665 directly pasted into an editor. Use -n to show them.
704 directly pasted into an editor. Use -n to show them.
666
705
667 Ranges of history can be indicated using the syntax:
706 Ranges of history can be indicated using the syntax:
668 4 : Line 4, current session
707 4 : Line 4, current session
669 4-6 : Lines 4-6, current session
708 4-6 : Lines 4-6, current session
670 243/1-5: Lines 1-5, session 243
709 243/1-5: Lines 1-5, session 243
671 ~2/7 : Line 7, session 2 before current
710 ~2/7 : Line 7, session 2 before current
672 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
711 ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
673 of 6 sessions ago.
712 of 6 sessions ago.
674 Multiple ranges can be entered, separated by spaces
713 Multiple ranges can be entered, separated by spaces
675
714
676 The same syntax is used by %macro, %save, %edit, %rerun
715 The same syntax is used by %macro, %save, %edit, %rerun
677
716
678 Options:
717 Options:
679
718
680 -n: print line numbers for each input.
719 -n: print line numbers for each input.
681 This feature is only available if numbered prompts are in use.
720 This feature is only available if numbered prompts are in use.
682
721
683 -o: also print outputs for each input.
722 -o: also print outputs for each input.
684
723
685 -p: print classic '>>>' python prompts before each input. This is useful
724 -p: print classic '>>>' python prompts before each input. This is useful
686 for making documentation, and in conjunction with -o, for producing
725 for making documentation, and in conjunction with -o, for producing
687 doctest-ready output.
726 doctest-ready output.
688
727
689 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
728 -r: (default) print the 'raw' history, i.e. the actual commands you typed.
690
729
691 -t: print the 'translated' history, as IPython understands it. IPython
730 -t: print the 'translated' history, as IPython understands it. IPython
692 filters your input and converts it all into valid Python source before
731 filters your input and converts it all into valid Python source before
693 executing it (things like magics or aliases are turned into function
732 executing it (things like magics or aliases are turned into function
694 calls, for example). With this option, you'll see the native history
733 calls, for example). With this option, you'll see the native history
695 instead of the user-entered version: '%cd /' will be seen as
734 instead of the user-entered version: '%cd /' will be seen as
696 'get_ipython().magic("%cd /")' instead of '%cd /'.
735 'get_ipython().magic("%cd /")' instead of '%cd /'.
697
736
698 -g: treat the arg as a pattern to grep for in (full) history.
737 -g: treat the arg as a pattern to grep for in (full) history.
699 This includes the saved history (almost all commands ever written).
738 This includes the saved history (almost all commands ever written).
700 Use '%hist -g' to show full saved history (may be very long).
739 Use '%hist -g' to show full saved history (may be very long).
701
740
702 -l: get the last n lines from all sessions. Specify n as a single arg, or
741 -l: get the last n lines from all sessions. Specify n as a single arg, or
703 the default is the last 10 lines.
742 the default is the last 10 lines.
704
743
705 -f FILENAME: instead of printing the output to the screen, redirect it to
744 -f FILENAME: instead of printing the output to the screen, redirect it to
706 the given file. The file is always overwritten, though IPython asks for
745 the given file. The file is always overwritten, though IPython asks for
707 confirmation first if it already exists.
746 confirmation first if it already exists.
708
747
709 Examples
748 Examples
710 --------
749 --------
711 ::
750 ::
712
751
713 In [6]: %hist -n 4 6
752 In [6]: %hist -n 4 6
714 4:a = 12
753 4:a = 12
715 5:print a**2
754 5:print a**2
716
755
717 """
756 """
718
757
719 if not self.shell.displayhook.do_full_cache:
758 if not self.shell.displayhook.do_full_cache:
720 print('This feature is only available if numbered prompts are in use.')
759 print('This feature is only available if numbered prompts are in use.')
721 return
760 return
722 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
761 opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
723
762
724 # For brevity
763 # For brevity
725 history_manager = self.shell.history_manager
764 history_manager = self.shell.history_manager
726
765
727 def _format_lineno(session, line):
766 def _format_lineno(session, line):
728 """Helper function to format line numbers properly."""
767 """Helper function to format line numbers properly."""
729 if session in (0, history_manager.session_number):
768 if session in (0, history_manager.session_number):
730 return str(line)
769 return str(line)
731 return "%s/%s" % (session, line)
770 return "%s/%s" % (session, line)
732
771
733 # Check if output to specific file was requested.
772 # Check if output to specific file was requested.
734 try:
773 try:
735 outfname = opts['f']
774 outfname = opts['f']
736 except KeyError:
775 except KeyError:
737 outfile = io.stdout # default
776 outfile = io.stdout # default
738 # We don't want to close stdout at the end!
777 # We don't want to close stdout at the end!
739 close_at_end = False
778 close_at_end = False
740 else:
779 else:
741 if os.path.exists(outfname):
780 if os.path.exists(outfname):
742 if not io.ask_yes_no("File %r exists. Overwrite?" % outfname):
781 if not io.ask_yes_no("File %r exists. Overwrite?" % outfname):
743 print('Aborting.')
782 print('Aborting.')
744 return
783 return
745
784
746 outfile = open(outfname,'w')
785 outfile = open(outfname,'w')
747 close_at_end = True
786 close_at_end = True
748
787
749 print_nums = 'n' in opts
788 print_nums = 'n' in opts
750 get_output = 'o' in opts
789 get_output = 'o' in opts
751 pyprompts = 'p' in opts
790 pyprompts = 'p' in opts
752 # Raw history is the default
791 # Raw history is the default
753 raw = not('t' in opts)
792 raw = not('t' in opts)
754
793
755 default_length = 40
794 default_length = 40
756 pattern = None
795 pattern = None
757
796
758 if 'g' in opts: # Glob search
797 if 'g' in opts: # Glob search
759 pattern = "*" + args + "*" if args else "*"
798 pattern = "*" + args + "*" if args else "*"
760 hist = history_manager.search(pattern, raw=raw, output=get_output)
799 hist = history_manager.search(pattern, raw=raw, output=get_output)
761 print_nums = True
800 print_nums = True
762 elif 'l' in opts: # Get 'tail'
801 elif 'l' in opts: # Get 'tail'
763 try:
802 try:
764 n = int(args)
803 n = int(args)
765 except ValueError, IndexError:
804 except ValueError, IndexError:
766 n = 10
805 n = 10
767 hist = history_manager.get_tail(n, raw=raw, output=get_output)
806 hist = history_manager.get_tail(n, raw=raw, output=get_output)
768 else:
807 else:
769 if args: # Get history by ranges
808 if args: # Get history by ranges
770 hist = history_manager.get_range_by_str(args, raw, get_output)
809 hist = history_manager.get_range_by_str(args, raw, get_output)
771 else: # Just get history for the current session
810 else: # Just get history for the current session
772 hist = history_manager.get_range(raw=raw, output=get_output)
811 hist = history_manager.get_range(raw=raw, output=get_output)
773
812
774 # We could be displaying the entire history, so let's not try to pull it
813 # We could be displaying the entire history, so let's not try to pull it
775 # into a list in memory. Anything that needs more space will just misalign.
814 # into a list in memory. Anything that needs more space will just misalign.
776 width = 4
815 width = 4
777
816
778 for session, lineno, inline in hist:
817 for session, lineno, inline in hist:
779 # Print user history with tabs expanded to 4 spaces. The GUI clients
818 # Print user history with tabs expanded to 4 spaces. The GUI clients
780 # use hard tabs for easier usability in auto-indented code, but we want
819 # use hard tabs for easier usability in auto-indented code, but we want
781 # to produce PEP-8 compliant history for safe pasting into an editor.
820 # to produce PEP-8 compliant history for safe pasting into an editor.
782 if get_output:
821 if get_output:
783 inline, output = inline
822 inline, output = inline
784 inline = inline.expandtabs(4).rstrip()
823 inline = inline.expandtabs(4).rstrip()
785
824
786 multiline = "\n" in inline
825 multiline = "\n" in inline
787 line_sep = '\n' if multiline else ' '
826 line_sep = '\n' if multiline else ' '
788 if print_nums:
827 if print_nums:
789 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
828 print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
790 line_sep), file=outfile, end='')
829 line_sep), file=outfile, end='')
791 if pyprompts:
830 if pyprompts:
792 print(">>> ", end="", file=outfile)
831 print(">>> ", end="", file=outfile)
793 if multiline:
832 if multiline:
794 inline = "\n... ".join(inline.splitlines()) + "\n..."
833 inline = "\n... ".join(inline.splitlines()) + "\n..."
795 print(inline, file=outfile)
834 print(inline, file=outfile)
796 if get_output and output:
835 if get_output and output:
797 print(output, file=outfile)
836 print(output, file=outfile)
798
837
799 if close_at_end:
838 if close_at_end:
800 outfile.close()
839 outfile.close()
801
840
802
841
803 def magic_rep(self, arg):
842 def magic_rep(self, arg):
804 r"""Repeat a command, or get command to input line for editing. %recall and
843 r"""Repeat a command, or get command to input line for editing. %recall and
805 %rep are equivalent.
844 %rep are equivalent.
806
845
807 - %recall (no arguments):
846 - %recall (no arguments):
808
847
809 Place a string version of last computation result (stored in the special '_'
848 Place a string version of last computation result (stored in the special '_'
810 variable) to the next input prompt. Allows you to create elaborate command
849 variable) to the next input prompt. Allows you to create elaborate command
811 lines without using copy-paste::
850 lines without using copy-paste::
812
851
813 In[1]: l = ["hei", "vaan"]
852 In[1]: l = ["hei", "vaan"]
814 In[2]: "".join(l)
853 In[2]: "".join(l)
815 Out[2]: heivaan
854 Out[2]: heivaan
816 In[3]: %rep
855 In[3]: %rep
817 In[4]: heivaan_ <== cursor blinking
856 In[4]: heivaan_ <== cursor blinking
818
857
819 %recall 45
858 %recall 45
820
859
821 Place history line 45 on the next input prompt. Use %hist to find
860 Place history line 45 on the next input prompt. Use %hist to find
822 out the number.
861 out the number.
823
862
824 %recall 1-4
863 %recall 1-4
825
864
826 Combine the specified lines into one cell, and place it on the next
865 Combine the specified lines into one cell, and place it on the next
827 input prompt. See %history for the slice syntax.
866 input prompt. See %history for the slice syntax.
828
867
829 %recall foo+bar
868 %recall foo+bar
830
869
831 If foo+bar can be evaluated in the user namespace, the result is
870 If foo+bar can be evaluated in the user namespace, the result is
832 placed at the next input prompt. Otherwise, the history is searched
871 placed at the next input prompt. Otherwise, the history is searched
833 for lines which contain that substring, and the most recent one is
872 for lines which contain that substring, and the most recent one is
834 placed at the next input prompt.
873 placed at the next input prompt.
835 """
874 """
836 if not arg: # Last output
875 if not arg: # Last output
837 self.set_next_input(str(self.shell.user_ns["_"]))
876 self.set_next_input(str(self.shell.user_ns["_"]))
838 return
877 return
839 # Get history range
878 # Get history range
840 histlines = self.history_manager.get_range_by_str(arg)
879 histlines = self.history_manager.get_range_by_str(arg)
841 cmd = "\n".join(x[2] for x in histlines)
880 cmd = "\n".join(x[2] for x in histlines)
842 if cmd:
881 if cmd:
843 self.set_next_input(cmd.rstrip())
882 self.set_next_input(cmd.rstrip())
844 return
883 return
845
884
846 try: # Variable in user namespace
885 try: # Variable in user namespace
847 cmd = str(eval(arg, self.shell.user_ns))
886 cmd = str(eval(arg, self.shell.user_ns))
848 except Exception: # Search for term in history
887 except Exception: # Search for term in history
849 histlines = self.history_manager.search("*"+arg+"*")
888 histlines = self.history_manager.search("*"+arg+"*")
850 for h in reversed([x[2] for x in histlines]):
889 for h in reversed([x[2] for x in histlines]):
851 if 'rep' in h:
890 if 'rep' in h:
852 continue
891 continue
853 self.set_next_input(h.rstrip())
892 self.set_next_input(h.rstrip())
854 return
893 return
855 else:
894 else:
856 self.set_next_input(cmd.rstrip())
895 self.set_next_input(cmd.rstrip())
857 print("Couldn't evaluate or find in history:", arg)
896 print("Couldn't evaluate or find in history:", arg)
858
897
859 def magic_rerun(self, parameter_s=''):
898 def magic_rerun(self, parameter_s=''):
860 """Re-run previous input
899 """Re-run previous input
861
900
862 By default, you can specify ranges of input history to be repeated
901 By default, you can specify ranges of input history to be repeated
863 (as with %history). With no arguments, it will repeat the last line.
902 (as with %history). With no arguments, it will repeat the last line.
864
903
865 Options:
904 Options:
866
905
867 -l <n> : Repeat the last n lines of input, not including the
906 -l <n> : Repeat the last n lines of input, not including the
868 current command.
907 current command.
869
908
870 -g foo : Repeat the most recent line which contains foo
909 -g foo : Repeat the most recent line which contains foo
871 """
910 """
872 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
911 opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
873 if "l" in opts: # Last n lines
912 if "l" in opts: # Last n lines
874 n = int(opts['l'])
913 n = int(opts['l'])
875 hist = self.history_manager.get_tail(n)
914 hist = self.history_manager.get_tail(n)
876 elif "g" in opts: # Search
915 elif "g" in opts: # Search
877 p = "*"+opts['g']+"*"
916 p = "*"+opts['g']+"*"
878 hist = list(self.history_manager.search(p))
917 hist = list(self.history_manager.search(p))
879 for l in reversed(hist):
918 for l in reversed(hist):
880 if "rerun" not in l[2]:
919 if "rerun" not in l[2]:
881 hist = [l] # The last match which isn't a %rerun
920 hist = [l] # The last match which isn't a %rerun
882 break
921 break
883 else:
922 else:
884 hist = [] # No matches except %rerun
923 hist = [] # No matches except %rerun
885 elif args: # Specify history ranges
924 elif args: # Specify history ranges
886 hist = self.history_manager.get_range_by_str(args)
925 hist = self.history_manager.get_range_by_str(args)
887 else: # Last line
926 else: # Last line
888 hist = self.history_manager.get_tail(1)
927 hist = self.history_manager.get_tail(1)
889 hist = [x[2] for x in hist]
928 hist = [x[2] for x in hist]
890 if not hist:
929 if not hist:
891 print("No lines in history match specification")
930 print("No lines in history match specification")
892 return
931 return
893 histlines = "\n".join(hist)
932 histlines = "\n".join(hist)
894 print("=== Executing: ===")
933 print("=== Executing: ===")
895 print(histlines)
934 print(histlines)
896 print("=== Output: ===")
935 print("=== Output: ===")
897 self.run_cell("\n".join(hist), store_history=False)
936 self.run_cell("\n".join(hist), store_history=False)
898
937
899
938
900 def init_ipython(ip):
939 def init_ipython(ip):
901 ip.define_magic("rep", magic_rep)
940 ip.define_magic("rep", magic_rep)
902 ip.define_magic("recall", magic_rep)
941 ip.define_magic("recall", magic_rep)
903 ip.define_magic("rerun", magic_rerun)
942 ip.define_magic("rerun", magic_rerun)
904 ip.define_magic("hist",magic_history) # Alternative name
943 ip.define_magic("hist",magic_history) # Alternative name
905 ip.define_magic("history",magic_history)
944 ip.define_magic("history",magic_history)
906
945
907 # XXX - ipy_completers are in quarantine, need to be updated to new apis
946 # XXX - ipy_completers are in quarantine, need to be updated to new apis
908 #import ipy_completers
947 #import ipy_completers
909 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
948 #ipy_completers.quick_completer('%hist' ,'-g -t -r -n')
@@ -1,452 +1,460 b''
1 """Tests for various magic functions.
1 """Tests for various magic functions.
2
2
3 Needs to be run by nose (to make ipython session available).
3 Needs to be run by nose (to make ipython session available).
4 """
4 """
5 from __future__ import absolute_import
5 from __future__ import absolute_import
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Imports
8 # Imports
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 import os
11 import os
12 import sys
12 import sys
13 import tempfile
13 import tempfile
14 import types
14 import types
15 from StringIO import StringIO
15 from StringIO import StringIO
16
16
17 import nose.tools as nt
17 import nose.tools as nt
18
18
19 from IPython.utils.path import get_long_path_name
19 from IPython.utils.path import get_long_path_name
20 from IPython.testing import decorators as dec
20 from IPython.testing import decorators as dec
21 from IPython.testing import tools as tt
21 from IPython.testing import tools as tt
22 from IPython.utils import py3compat
22 from IPython.utils import py3compat
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Test functions begin
25 # Test functions begin
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 def test_rehashx():
27 def test_rehashx():
28 # clear up everything
28 # clear up everything
29 _ip = get_ipython()
29 _ip = get_ipython()
30 _ip.alias_manager.alias_table.clear()
30 _ip.alias_manager.alias_table.clear()
31 del _ip.db['syscmdlist']
31 del _ip.db['syscmdlist']
32
32
33 _ip.magic('rehashx')
33 _ip.magic('rehashx')
34 # Practically ALL ipython development systems will have more than 10 aliases
34 # Practically ALL ipython development systems will have more than 10 aliases
35
35
36 yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
36 yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
37 for key, val in _ip.alias_manager.alias_table.iteritems():
37 for key, val in _ip.alias_manager.alias_table.iteritems():
38 # we must strip dots from alias names
38 # we must strip dots from alias names
39 nt.assert_true('.' not in key)
39 nt.assert_true('.' not in key)
40
40
41 # rehashx must fill up syscmdlist
41 # rehashx must fill up syscmdlist
42 scoms = _ip.db['syscmdlist']
42 scoms = _ip.db['syscmdlist']
43 yield (nt.assert_true, len(scoms) > 10)
43 yield (nt.assert_true, len(scoms) > 10)
44
44
45
45
46 def test_magic_parse_options():
46 def test_magic_parse_options():
47 """Test that we don't mangle paths when parsing magic options."""
47 """Test that we don't mangle paths when parsing magic options."""
48 ip = get_ipython()
48 ip = get_ipython()
49 path = 'c:\\x'
49 path = 'c:\\x'
50 opts = ip.parse_options('-f %s' % path,'f:')[0]
50 opts = ip.parse_options('-f %s' % path,'f:')[0]
51 # argv splitting is os-dependent
51 # argv splitting is os-dependent
52 if os.name == 'posix':
52 if os.name == 'posix':
53 expected = 'c:x'
53 expected = 'c:x'
54 else:
54 else:
55 expected = path
55 expected = path
56 nt.assert_equals(opts['f'], expected)
56 nt.assert_equals(opts['f'], expected)
57
57
58
58
59 @dec.skip_without('sqlite3')
59 def doctest_hist_f():
60 def doctest_hist_f():
60 """Test %hist -f with temporary filename.
61 """Test %hist -f with temporary filename.
61
62
62 In [9]: import tempfile
63 In [9]: import tempfile
63
64
64 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
65 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
65
66
66 In [11]: %hist -nl -f $tfile 3
67 In [11]: %hist -nl -f $tfile 3
67
68
68 In [13]: import os; os.unlink(tfile)
69 In [13]: import os; os.unlink(tfile)
69 """
70 """
70
71
71
72
73 @dec.skip_without('sqlite3')
72 def doctest_hist_r():
74 def doctest_hist_r():
73 """Test %hist -r
75 """Test %hist -r
74
76
75 XXX - This test is not recording the output correctly. For some reason, in
77 XXX - This test is not recording the output correctly. For some reason, in
76 testing mode the raw history isn't getting populated. No idea why.
78 testing mode the raw history isn't getting populated. No idea why.
77 Disabling the output checking for now, though at least we do run it.
79 Disabling the output checking for now, though at least we do run it.
78
80
79 In [1]: 'hist' in _ip.lsmagic()
81 In [1]: 'hist' in _ip.lsmagic()
80 Out[1]: True
82 Out[1]: True
81
83
82 In [2]: x=1
84 In [2]: x=1
83
85
84 In [3]: %hist -rl 2
86 In [3]: %hist -rl 2
85 x=1 # random
87 x=1 # random
86 %hist -r 2
88 %hist -r 2
87 """
89 """
88
90
91
92 @dec.skip_without('sqlite3')
89 def doctest_hist_op():
93 def doctest_hist_op():
90 """Test %hist -op
94 """Test %hist -op
91
95
92 In [1]: class b(float):
96 In [1]: class b(float):
93 ...: pass
97 ...: pass
94 ...:
98 ...:
95
99
96 In [2]: class s(object):
100 In [2]: class s(object):
97 ...: def __str__(self):
101 ...: def __str__(self):
98 ...: return 's'
102 ...: return 's'
99 ...:
103 ...:
100
104
101 In [3]:
105 In [3]:
102
106
103 In [4]: class r(b):
107 In [4]: class r(b):
104 ...: def __repr__(self):
108 ...: def __repr__(self):
105 ...: return 'r'
109 ...: return 'r'
106 ...:
110 ...:
107
111
108 In [5]: class sr(s,r): pass
112 In [5]: class sr(s,r): pass
109 ...:
113 ...:
110
114
111 In [6]:
115 In [6]:
112
116
113 In [7]: bb=b()
117 In [7]: bb=b()
114
118
115 In [8]: ss=s()
119 In [8]: ss=s()
116
120
117 In [9]: rr=r()
121 In [9]: rr=r()
118
122
119 In [10]: ssrr=sr()
123 In [10]: ssrr=sr()
120
124
121 In [11]: 4.5
125 In [11]: 4.5
122 Out[11]: 4.5
126 Out[11]: 4.5
123
127
124 In [12]: str(ss)
128 In [12]: str(ss)
125 Out[12]: 's'
129 Out[12]: 's'
126
130
127 In [13]:
131 In [13]:
128
132
129 In [14]: %hist -op
133 In [14]: %hist -op
130 >>> class b:
134 >>> class b:
131 ... pass
135 ... pass
132 ...
136 ...
133 >>> class s(b):
137 >>> class s(b):
134 ... def __str__(self):
138 ... def __str__(self):
135 ... return 's'
139 ... return 's'
136 ...
140 ...
137 >>>
141 >>>
138 >>> class r(b):
142 >>> class r(b):
139 ... def __repr__(self):
143 ... def __repr__(self):
140 ... return 'r'
144 ... return 'r'
141 ...
145 ...
142 >>> class sr(s,r): pass
146 >>> class sr(s,r): pass
143 >>>
147 >>>
144 >>> bb=b()
148 >>> bb=b()
145 >>> ss=s()
149 >>> ss=s()
146 >>> rr=r()
150 >>> rr=r()
147 >>> ssrr=sr()
151 >>> ssrr=sr()
148 >>> 4.5
152 >>> 4.5
149 4.5
153 4.5
150 >>> str(ss)
154 >>> str(ss)
151 's'
155 's'
152 >>>
156 >>>
153 """
157 """
154
158
159
160 @dec.skip_without('sqlite3')
155 def test_macro():
161 def test_macro():
156 ip = get_ipython()
162 ip = get_ipython()
157 ip.history_manager.reset() # Clear any existing history.
163 ip.history_manager.reset() # Clear any existing history.
158 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
164 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
159 for i, cmd in enumerate(cmds, start=1):
165 for i, cmd in enumerate(cmds, start=1):
160 ip.history_manager.store_inputs(i, cmd)
166 ip.history_manager.store_inputs(i, cmd)
161 ip.magic("macro test 1-3")
167 ip.magic("macro test 1-3")
162 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
168 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
163
169
164 # List macros.
170 # List macros.
165 assert "test" in ip.magic("macro")
171 assert "test" in ip.magic("macro")
166
172
173
174 @dec.skip_without('sqlite3')
167 def test_macro_run():
175 def test_macro_run():
168 """Test that we can run a multi-line macro successfully."""
176 """Test that we can run a multi-line macro successfully."""
169 ip = get_ipython()
177 ip = get_ipython()
170 ip.history_manager.reset()
178 ip.history_manager.reset()
171 cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
179 cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
172 "%macro test 2-3"]
180 "%macro test 2-3"]
173 for cmd in cmds:
181 for cmd in cmds:
174 ip.run_cell(cmd, store_history=True)
182 ip.run_cell(cmd, store_history=True)
175 nt.assert_equal(ip.user_ns["test"].value,
183 nt.assert_equal(ip.user_ns["test"].value,
176 py3compat.doctest_refactor_print("a+=1\nprint a\n"))
184 py3compat.doctest_refactor_print("a+=1\nprint a\n"))
177 with tt.AssertPrints("12"):
185 with tt.AssertPrints("12"):
178 ip.run_cell("test")
186 ip.run_cell("test")
179 with tt.AssertPrints("13"):
187 with tt.AssertPrints("13"):
180 ip.run_cell("test")
188 ip.run_cell("test")
181
189
182
190
183 # XXX failing for now, until we get clearcmd out of quarantine. But we should
191 # XXX failing for now, until we get clearcmd out of quarantine. But we should
184 # fix this and revert the skip to happen only if numpy is not around.
192 # fix this and revert the skip to happen only if numpy is not around.
185 #@dec.skipif_not_numpy
193 #@dec.skipif_not_numpy
186 @dec.skip_known_failure
194 @dec.skip_known_failure
187 def test_numpy_clear_array_undec():
195 def test_numpy_clear_array_undec():
188 from IPython.extensions import clearcmd
196 from IPython.extensions import clearcmd
189
197
190 _ip.ex('import numpy as np')
198 _ip.ex('import numpy as np')
191 _ip.ex('a = np.empty(2)')
199 _ip.ex('a = np.empty(2)')
192 yield (nt.assert_true, 'a' in _ip.user_ns)
200 yield (nt.assert_true, 'a' in _ip.user_ns)
193 _ip.magic('clear array')
201 _ip.magic('clear array')
194 yield (nt.assert_false, 'a' in _ip.user_ns)
202 yield (nt.assert_false, 'a' in _ip.user_ns)
195
203
196
204
197 # Multiple tests for clipboard pasting
205 # Multiple tests for clipboard pasting
198 @dec.parametric
206 @dec.parametric
199 def test_paste():
207 def test_paste():
200 _ip = get_ipython()
208 _ip = get_ipython()
201 def paste(txt, flags='-q'):
209 def paste(txt, flags='-q'):
202 """Paste input text, by default in quiet mode"""
210 """Paste input text, by default in quiet mode"""
203 hooks.clipboard_get = lambda : txt
211 hooks.clipboard_get = lambda : txt
204 _ip.magic('paste '+flags)
212 _ip.magic('paste '+flags)
205
213
206 # Inject fake clipboard hook but save original so we can restore it later
214 # Inject fake clipboard hook but save original so we can restore it later
207 hooks = _ip.hooks
215 hooks = _ip.hooks
208 user_ns = _ip.user_ns
216 user_ns = _ip.user_ns
209 original_clip = hooks.clipboard_get
217 original_clip = hooks.clipboard_get
210
218
211 try:
219 try:
212 # Run tests with fake clipboard function
220 # Run tests with fake clipboard function
213 user_ns.pop('x', None)
221 user_ns.pop('x', None)
214 paste('x=1')
222 paste('x=1')
215 yield nt.assert_equal(user_ns['x'], 1)
223 yield nt.assert_equal(user_ns['x'], 1)
216
224
217 user_ns.pop('x', None)
225 user_ns.pop('x', None)
218 paste('>>> x=2')
226 paste('>>> x=2')
219 yield nt.assert_equal(user_ns['x'], 2)
227 yield nt.assert_equal(user_ns['x'], 2)
220
228
221 paste("""
229 paste("""
222 >>> x = [1,2,3]
230 >>> x = [1,2,3]
223 >>> y = []
231 >>> y = []
224 >>> for i in x:
232 >>> for i in x:
225 ... y.append(i**2)
233 ... y.append(i**2)
226 ...
234 ...
227 """)
235 """)
228 yield nt.assert_equal(user_ns['x'], [1,2,3])
236 yield nt.assert_equal(user_ns['x'], [1,2,3])
229 yield nt.assert_equal(user_ns['y'], [1,4,9])
237 yield nt.assert_equal(user_ns['y'], [1,4,9])
230
238
231 # Now, test that paste -r works
239 # Now, test that paste -r works
232 user_ns.pop('x', None)
240 user_ns.pop('x', None)
233 yield nt.assert_false('x' in user_ns)
241 yield nt.assert_false('x' in user_ns)
234 _ip.magic('paste -r')
242 _ip.magic('paste -r')
235 yield nt.assert_equal(user_ns['x'], [1,2,3])
243 yield nt.assert_equal(user_ns['x'], [1,2,3])
236
244
237 # Also test paste echoing, by temporarily faking the writer
245 # Also test paste echoing, by temporarily faking the writer
238 w = StringIO()
246 w = StringIO()
239 writer = _ip.write
247 writer = _ip.write
240 _ip.write = w.write
248 _ip.write = w.write
241 code = """
249 code = """
242 a = 100
250 a = 100
243 b = 200"""
251 b = 200"""
244 try:
252 try:
245 paste(code,'')
253 paste(code,'')
246 out = w.getvalue()
254 out = w.getvalue()
247 finally:
255 finally:
248 _ip.write = writer
256 _ip.write = writer
249 yield nt.assert_equal(user_ns['a'], 100)
257 yield nt.assert_equal(user_ns['a'], 100)
250 yield nt.assert_equal(user_ns['b'], 200)
258 yield nt.assert_equal(user_ns['b'], 200)
251 yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
259 yield nt.assert_equal(out, code+"\n## -- End pasted text --\n")
252
260
253 finally:
261 finally:
254 # Restore original hook
262 # Restore original hook
255 hooks.clipboard_get = original_clip
263 hooks.clipboard_get = original_clip
256
264
257
265
258 def test_time():
266 def test_time():
259 _ip.magic('time None')
267 _ip.magic('time None')
260
268
261
269
262 @py3compat.doctest_refactor_print
270 @py3compat.doctest_refactor_print
263 def doctest_time():
271 def doctest_time():
264 """
272 """
265 In [10]: %time None
273 In [10]: %time None
266 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
274 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
267 Wall time: 0.00 s
275 Wall time: 0.00 s
268
276
269 In [11]: def f(kmjy):
277 In [11]: def f(kmjy):
270 ....: %time print 2*kmjy
278 ....: %time print 2*kmjy
271
279
272 In [12]: f(3)
280 In [12]: f(3)
273 6
281 6
274 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
282 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
275 Wall time: 0.00 s
283 Wall time: 0.00 s
276 """
284 """
277
285
278
286
279 def test_doctest_mode():
287 def test_doctest_mode():
280 "Toggle doctest_mode twice, it should be a no-op and run without error"
288 "Toggle doctest_mode twice, it should be a no-op and run without error"
281 _ip.magic('doctest_mode')
289 _ip.magic('doctest_mode')
282 _ip.magic('doctest_mode')
290 _ip.magic('doctest_mode')
283
291
284
292
285 def test_parse_options():
293 def test_parse_options():
286 """Tests for basic options parsing in magics."""
294 """Tests for basic options parsing in magics."""
287 # These are only the most minimal of tests, more should be added later. At
295 # These are only the most minimal of tests, more should be added later. At
288 # the very least we check that basic text/unicode calls work OK.
296 # the very least we check that basic text/unicode calls work OK.
289 nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
297 nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
290 nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
298 nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
291
299
292
300
293 def test_dirops():
301 def test_dirops():
294 """Test various directory handling operations."""
302 """Test various directory handling operations."""
295 # curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
303 # curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
296 curpath = os.getcwdu
304 curpath = os.getcwdu
297 startdir = os.getcwdu()
305 startdir = os.getcwdu()
298 ipdir = _ip.ipython_dir
306 ipdir = _ip.ipython_dir
299 try:
307 try:
300 _ip.magic('cd "%s"' % ipdir)
308 _ip.magic('cd "%s"' % ipdir)
301 nt.assert_equal(curpath(), ipdir)
309 nt.assert_equal(curpath(), ipdir)
302 _ip.magic('cd -')
310 _ip.magic('cd -')
303 nt.assert_equal(curpath(), startdir)
311 nt.assert_equal(curpath(), startdir)
304 _ip.magic('pushd "%s"' % ipdir)
312 _ip.magic('pushd "%s"' % ipdir)
305 nt.assert_equal(curpath(), ipdir)
313 nt.assert_equal(curpath(), ipdir)
306 _ip.magic('popd')
314 _ip.magic('popd')
307 nt.assert_equal(curpath(), startdir)
315 nt.assert_equal(curpath(), startdir)
308 finally:
316 finally:
309 os.chdir(startdir)
317 os.chdir(startdir)
310
318
311
319
312 def check_cpaste(code, should_fail=False):
320 def check_cpaste(code, should_fail=False):
313 """Execute code via 'cpaste' and ensure it was executed, unless
321 """Execute code via 'cpaste' and ensure it was executed, unless
314 should_fail is set.
322 should_fail is set.
315 """
323 """
316 _ip.user_ns['code_ran'] = False
324 _ip.user_ns['code_ran'] = False
317
325
318 src = StringIO()
326 src = StringIO()
319 src.encoding = None # IPython expects stdin to have an encoding attribute
327 src.encoding = None # IPython expects stdin to have an encoding attribute
320 src.write('\n')
328 src.write('\n')
321 src.write(code)
329 src.write(code)
322 src.write('\n--\n')
330 src.write('\n--\n')
323 src.seek(0)
331 src.seek(0)
324
332
325 stdin_save = sys.stdin
333 stdin_save = sys.stdin
326 sys.stdin = src
334 sys.stdin = src
327
335
328 try:
336 try:
329 context = tt.AssertPrints if should_fail else tt.AssertNotPrints
337 context = tt.AssertPrints if should_fail else tt.AssertNotPrints
330 with context("Traceback (most recent call last)"):
338 with context("Traceback (most recent call last)"):
331 _ip.magic('cpaste')
339 _ip.magic('cpaste')
332
340
333 if not should_fail:
341 if not should_fail:
334 assert _ip.user_ns['code_ran']
342 assert _ip.user_ns['code_ran']
335 finally:
343 finally:
336 sys.stdin = stdin_save
344 sys.stdin = stdin_save
337
345
338
346
339 def test_cpaste():
347 def test_cpaste():
340 """Test cpaste magic"""
348 """Test cpaste magic"""
341
349
342 def run():
350 def run():
343 """Marker function: sets a flag when executed.
351 """Marker function: sets a flag when executed.
344 """
352 """
345 _ip.user_ns['code_ran'] = True
353 _ip.user_ns['code_ran'] = True
346 return 'run' # return string so '+ run()' doesn't result in success
354 return 'run' # return string so '+ run()' doesn't result in success
347
355
348 tests = {'pass': ["> > > run()",
356 tests = {'pass': ["> > > run()",
349 ">>> > run()",
357 ">>> > run()",
350 "+++ run()",
358 "+++ run()",
351 "++ run()",
359 "++ run()",
352 " >>> run()"],
360 " >>> run()"],
353
361
354 'fail': ["+ + run()",
362 'fail': ["+ + run()",
355 " ++ run()"]}
363 " ++ run()"]}
356
364
357 _ip.user_ns['run'] = run
365 _ip.user_ns['run'] = run
358
366
359 for code in tests['pass']:
367 for code in tests['pass']:
360 check_cpaste(code)
368 check_cpaste(code)
361
369
362 for code in tests['fail']:
370 for code in tests['fail']:
363 check_cpaste(code, should_fail=True)
371 check_cpaste(code, should_fail=True)
364
372
365 def test_xmode():
373 def test_xmode():
366 # Calling xmode three times should be a no-op
374 # Calling xmode three times should be a no-op
367 xmode = _ip.InteractiveTB.mode
375 xmode = _ip.InteractiveTB.mode
368 for i in range(3):
376 for i in range(3):
369 _ip.magic("xmode")
377 _ip.magic("xmode")
370 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
378 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
371
379
372 def test_reset_hard():
380 def test_reset_hard():
373 monitor = []
381 monitor = []
374 class A(object):
382 class A(object):
375 def __del__(self):
383 def __del__(self):
376 monitor.append(1)
384 monitor.append(1)
377 def __repr__(self):
385 def __repr__(self):
378 return "<A instance>"
386 return "<A instance>"
379
387
380 _ip.user_ns["a"] = A()
388 _ip.user_ns["a"] = A()
381 _ip.run_cell("a")
389 _ip.run_cell("a")
382
390
383 nt.assert_equal(monitor, [])
391 nt.assert_equal(monitor, [])
384 _ip.magic_reset("-f")
392 _ip.magic_reset("-f")
385 nt.assert_equal(monitor, [1])
393 nt.assert_equal(monitor, [1])
386
394
387 class TestXdel(tt.TempFileMixin):
395 class TestXdel(tt.TempFileMixin):
388 def test_xdel(self):
396 def test_xdel(self):
389 """Test that references from %run are cleared by xdel."""
397 """Test that references from %run are cleared by xdel."""
390 src = ("class A(object):\n"
398 src = ("class A(object):\n"
391 " monitor = []\n"
399 " monitor = []\n"
392 " def __del__(self):\n"
400 " def __del__(self):\n"
393 " self.monitor.append(1)\n"
401 " self.monitor.append(1)\n"
394 "a = A()\n")
402 "a = A()\n")
395 self.mktmp(src)
403 self.mktmp(src)
396 # %run creates some hidden references...
404 # %run creates some hidden references...
397 _ip.magic("run %s" % self.fname)
405 _ip.magic("run %s" % self.fname)
398 # ... as does the displayhook.
406 # ... as does the displayhook.
399 _ip.run_cell("a")
407 _ip.run_cell("a")
400
408
401 monitor = _ip.user_ns["A"].monitor
409 monitor = _ip.user_ns["A"].monitor
402 nt.assert_equal(monitor, [])
410 nt.assert_equal(monitor, [])
403
411
404 _ip.magic("xdel a")
412 _ip.magic("xdel a")
405
413
406 # Check that a's __del__ method has been called.
414 # Check that a's __del__ method has been called.
407 nt.assert_equal(monitor, [1])
415 nt.assert_equal(monitor, [1])
408
416
409 def doctest_who():
417 def doctest_who():
410 """doctest for %who
418 """doctest for %who
411
419
412 In [1]: %reset -f
420 In [1]: %reset -f
413
421
414 In [2]: alpha = 123
422 In [2]: alpha = 123
415
423
416 In [3]: beta = 'beta'
424 In [3]: beta = 'beta'
417
425
418 In [4]: %who int
426 In [4]: %who int
419 alpha
427 alpha
420
428
421 In [5]: %who str
429 In [5]: %who str
422 beta
430 beta
423
431
424 In [6]: %whos
432 In [6]: %whos
425 Variable Type Data/Info
433 Variable Type Data/Info
426 ----------------------------
434 ----------------------------
427 alpha int 123
435 alpha int 123
428 beta str beta
436 beta str beta
429
437
430 In [7]: %who_ls
438 In [7]: %who_ls
431 Out[7]: ['alpha', 'beta']
439 Out[7]: ['alpha', 'beta']
432 """
440 """
433
441
434 @py3compat.u_format
442 @py3compat.u_format
435 def doctest_precision():
443 def doctest_precision():
436 """doctest for %precision
444 """doctest for %precision
437
445
438 In [1]: f = get_ipython().shell.display_formatter.formatters['text/plain']
446 In [1]: f = get_ipython().shell.display_formatter.formatters['text/plain']
439
447
440 In [2]: %precision 5
448 In [2]: %precision 5
441 Out[2]: {u}'%.5f'
449 Out[2]: {u}'%.5f'
442
450
443 In [3]: f.float_format
451 In [3]: f.float_format
444 Out[3]: {u}'%.5f'
452 Out[3]: {u}'%.5f'
445
453
446 In [4]: %precision %e
454 In [4]: %precision %e
447 Out[4]: {u}'%e'
455 Out[4]: {u}'%e'
448
456
449 In [5]: f(3.1415927)
457 In [5]: f(3.1415927)
450 Out[5]: {u}'3.141593e+00'
458 Out[5]: {u}'3.141593e+00'
451 """
459 """
452
460
@@ -1,210 +1,218 b''
1 """Tests for code execution (%run and related), which is particularly tricky.
1 """Tests for code execution (%run and related), which is particularly tricky.
2
2
3 Because of how %run manages namespaces, and the fact that we are trying here to
3 Because of how %run manages namespaces, and the fact that we are trying here to
4 verify subtle object deletion and reference counting issues, the %run tests
4 verify subtle object deletion and reference counting issues, the %run tests
5 will be kept in this separate file. This makes it easier to aggregate in one
5 will be kept in this separate file. This makes it easier to aggregate in one
6 place the tricks needed to handle it; most other magics are much easier to test
6 place the tricks needed to handle it; most other magics are much easier to test
7 and we do so in a common test_magic file.
7 and we do so in a common test_magic file.
8 """
8 """
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 import os
15 import os
16 import sys
16 import sys
17 import tempfile
17 import tempfile
18
18
19 import nose.tools as nt
19 import nose.tools as nt
20 from nose import SkipTest
20 from nose import SkipTest
21
21
22 from IPython.testing import decorators as dec
22 from IPython.testing import decorators as dec
23 from IPython.testing import tools as tt
23 from IPython.testing import tools as tt
24 from IPython.utils import py3compat
24 from IPython.utils import py3compat
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Test functions begin
27 # Test functions begin
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29
29
30 def doctest_refbug():
30 def doctest_refbug():
31 """Very nasty problem with references held by multiple runs of a script.
31 """Very nasty problem with references held by multiple runs of a script.
32 See: https://github.com/ipython/ipython/issues/141
32 See: https://github.com/ipython/ipython/issues/141
33
33
34 In [1]: _ip.clear_main_mod_cache()
34 In [1]: _ip.clear_main_mod_cache()
35 # random
35 # random
36
36
37 In [2]: %run refbug
37 In [2]: %run refbug
38
38
39 In [3]: call_f()
39 In [3]: call_f()
40 lowercased: hello
40 lowercased: hello
41
41
42 In [4]: %run refbug
42 In [4]: %run refbug
43
43
44 In [5]: call_f()
44 In [5]: call_f()
45 lowercased: hello
45 lowercased: hello
46 lowercased: hello
46 lowercased: hello
47 """
47 """
48
48
49
49
50 def doctest_run_builtins():
50 def doctest_run_builtins():
51 r"""Check that %run doesn't damage __builtins__.
51 r"""Check that %run doesn't damage __builtins__.
52
52
53 In [1]: import tempfile
53 In [1]: import tempfile
54
54
55 In [2]: bid1 = id(__builtins__)
55 In [2]: bid1 = id(__builtins__)
56
56
57 In [3]: fname = tempfile.mkstemp('.py')[1]
57 In [3]: fname = tempfile.mkstemp('.py')[1]
58
58
59 In [3]: f = open(fname,'w')
59 In [3]: f = open(fname,'w')
60
60
61 In [4]: dummy= f.write('pass\n')
61 In [4]: dummy= f.write('pass\n')
62
62
63 In [5]: f.flush()
63 In [5]: f.flush()
64
64
65 In [6]: t1 = type(__builtins__)
65 In [6]: t1 = type(__builtins__)
66
66
67 In [7]: %run $fname
67 In [7]: %run $fname
68
68
69 In [7]: f.close()
69 In [7]: f.close()
70
70
71 In [8]: bid2 = id(__builtins__)
71 In [8]: bid2 = id(__builtins__)
72
72
73 In [9]: t2 = type(__builtins__)
73 In [9]: t2 = type(__builtins__)
74
74
75 In [10]: t1 == t2
75 In [10]: t1 == t2
76 Out[10]: True
76 Out[10]: True
77
77
78 In [10]: bid1 == bid2
78 In [10]: bid1 == bid2
79 Out[10]: True
79 Out[10]: True
80
80
81 In [12]: try:
81 In [12]: try:
82 ....: os.unlink(fname)
82 ....: os.unlink(fname)
83 ....: except:
83 ....: except:
84 ....: pass
84 ....: pass
85 ....:
85 ....:
86 """
86 """
87
87
88 @py3compat.doctest_refactor_print
88 @py3compat.doctest_refactor_print
89 def doctest_reset_del():
89 def doctest_reset_del():
90 """Test that resetting doesn't cause errors in __del__ methods.
90 """Test that resetting doesn't cause errors in __del__ methods.
91
91
92 In [2]: class A(object):
92 In [2]: class A(object):
93 ...: def __del__(self):
93 ...: def __del__(self):
94 ...: print str("Hi")
94 ...: print str("Hi")
95 ...:
95 ...:
96
96
97 In [3]: a = A()
97 In [3]: a = A()
98
98
99 In [4]: get_ipython().reset()
99 In [4]: get_ipython().reset()
100 Hi
100 Hi
101
101
102 In [5]: 1+1
102 In [5]: 1+1
103 Out[5]: 2
103 Out[5]: 2
104 """
104 """
105
105
106 # For some tests, it will be handy to organize them in a class with a common
106 # For some tests, it will be handy to organize them in a class with a common
107 # setup that makes a temp file
107 # setup that makes a temp file
108
108
109 class TestMagicRunPass(tt.TempFileMixin):
109 class TestMagicRunPass(tt.TempFileMixin):
110
110
111 def setup(self):
111 def setup(self):
112 """Make a valid python temp file."""
112 """Make a valid python temp file."""
113 self.mktmp('pass\n')
113 self.mktmp('pass\n')
114
114
115 def run_tmpfile(self):
115 def run_tmpfile(self):
116 _ip = get_ipython()
116 _ip = get_ipython()
117 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
117 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
118 # See below and ticket https://bugs.launchpad.net/bugs/366353
118 # See below and ticket https://bugs.launchpad.net/bugs/366353
119 _ip.magic('run %s' % self.fname)
119 _ip.magic('run %s' % self.fname)
120
120
121 def test_builtins_id(self):
121 def test_builtins_id(self):
122 """Check that %run doesn't damage __builtins__ """
122 """Check that %run doesn't damage __builtins__ """
123 _ip = get_ipython()
123 _ip = get_ipython()
124 # Test that the id of __builtins__ is not modified by %run
124 # Test that the id of __builtins__ is not modified by %run
125 bid1 = id(_ip.user_ns['__builtins__'])
125 bid1 = id(_ip.user_ns['__builtins__'])
126 self.run_tmpfile()
126 self.run_tmpfile()
127 bid2 = id(_ip.user_ns['__builtins__'])
127 bid2 = id(_ip.user_ns['__builtins__'])
128 tt.assert_equals(bid1, bid2)
128 tt.assert_equals(bid1, bid2)
129
129
130 def test_builtins_type(self):
130 def test_builtins_type(self):
131 """Check that the type of __builtins__ doesn't change with %run.
131 """Check that the type of __builtins__ doesn't change with %run.
132
132
133 However, the above could pass if __builtins__ was already modified to
133 However, the above could pass if __builtins__ was already modified to
134 be a dict (it should be a module) by a previous use of %run. So we
134 be a dict (it should be a module) by a previous use of %run. So we
135 also check explicitly that it really is a module:
135 also check explicitly that it really is a module:
136 """
136 """
137 _ip = get_ipython()
137 _ip = get_ipython()
138 self.run_tmpfile()
138 self.run_tmpfile()
139 tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
139 tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
140
140
141 def test_prompts(self):
141 def test_prompts(self):
142 """Test that prompts correctly generate after %run"""
142 """Test that prompts correctly generate after %run"""
143 self.run_tmpfile()
143 self.run_tmpfile()
144 _ip = get_ipython()
144 _ip = get_ipython()
145 p2 = str(_ip.displayhook.prompt2).strip()
145 p2 = str(_ip.displayhook.prompt2).strip()
146 nt.assert_equals(p2[:3], '...')
146 nt.assert_equals(p2[:3], '...')
147
147
148
148
149 class TestMagicRunSimple(tt.TempFileMixin):
149 class TestMagicRunSimple(tt.TempFileMixin):
150
150
151 def test_simpledef(self):
151 def test_simpledef(self):
152 """Test that simple class definitions work."""
152 """Test that simple class definitions work."""
153 src = ("class foo: pass\n"
153 src = ("class foo: pass\n"
154 "def f(): return foo()")
154 "def f(): return foo()")
155 self.mktmp(src)
155 self.mktmp(src)
156 _ip.magic('run %s' % self.fname)
156 _ip.magic('run %s' % self.fname)
157 _ip.run_cell('t = isinstance(f(), foo)')
157 _ip.run_cell('t = isinstance(f(), foo)')
158 nt.assert_true(_ip.user_ns['t'])
158 nt.assert_true(_ip.user_ns['t'])
159
159
160 def test_obj_del(self):
160 def test_obj_del(self):
161 """Test that object's __del__ methods are called on exit."""
161 """Test that object's __del__ methods are called on exit."""
162 if sys.platform == 'win32':
162 if sys.platform == 'win32':
163 try:
163 try:
164 import win32api
164 import win32api
165 except ImportError:
165 except ImportError:
166 raise SkipTest("Test requires pywin32")
166 raise SkipTest("Test requires pywin32")
167 src = ("class A(object):\n"
167 src = ("class A(object):\n"
168 " def __del__(self):\n"
168 " def __del__(self):\n"
169 " print 'object A deleted'\n"
169 " print 'object A deleted'\n"
170 "a = A()\n")
170 "a = A()\n")
171 self.mktmp(py3compat.doctest_refactor_print(src))
171 self.mktmp(py3compat.doctest_refactor_print(src))
172 tt.ipexec_validate(self.fname, 'object A deleted')
172 if dec.module_not_available('sqlite3'):
173 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
174 else:
175 err = None
176 tt.ipexec_validate(self.fname, 'object A deleted', err)
173
177
174 @dec.skip_known_failure
178 @dec.skip_known_failure
175 def test_aggressive_namespace_cleanup(self):
179 def test_aggressive_namespace_cleanup(self):
176 """Test that namespace cleanup is not too aggressive GH-238
180 """Test that namespace cleanup is not too aggressive GH-238
177
181
178 Returning from another run magic deletes the namespace"""
182 Returning from another run magic deletes the namespace"""
179 # see ticket https://github.com/ipython/ipython/issues/238
183 # see ticket https://github.com/ipython/ipython/issues/238
180 class secondtmp(tt.TempFileMixin): pass
184 class secondtmp(tt.TempFileMixin): pass
181 empty = secondtmp()
185 empty = secondtmp()
182 empty.mktmp('')
186 empty.mktmp('')
183 src = ("ip = get_ipython()\n"
187 src = ("ip = get_ipython()\n"
184 "for i in range(5):\n"
188 "for i in range(5):\n"
185 " try:\n"
189 " try:\n"
186 " ip.magic('run %s')\n"
190 " ip.magic('run %s')\n"
187 " except NameError, e:\n"
191 " except NameError, e:\n"
188 " print i;break\n" % empty.fname)
192 " print i;break\n" % empty.fname)
189 self.mktmp(py3compat.doctest_refactor_print(src))
193 self.mktmp(py3compat.doctest_refactor_print(src))
190 _ip.magic('run %s' % self.fname)
194 _ip.magic('run %s' % self.fname)
191 _ip.run_cell('ip == get_ipython()')
195 _ip.run_cell('ip == get_ipython()')
192 tt.assert_equals(_ip.user_ns['i'], 5)
196 tt.assert_equals(_ip.user_ns['i'], 5)
193
197
194 @dec.skip_win32
198 @dec.skip_win32
195 def test_tclass(self):
199 def test_tclass(self):
196 mydir = os.path.dirname(__file__)
200 mydir = os.path.dirname(__file__)
197 tc = os.path.join(mydir, 'tclass')
201 tc = os.path.join(mydir, 'tclass')
198 src = ("%%run '%s' C-first\n"
202 src = ("%%run '%s' C-first\n"
199 "%%run '%s' C-second\n"
203 "%%run '%s' C-second\n"
200 "%%run '%s' C-third\n") % (tc, tc, tc)
204 "%%run '%s' C-third\n") % (tc, tc, tc)
201 self.mktmp(src, '.ipy')
205 self.mktmp(src, '.ipy')
202 out = """\
206 out = """\
203 ARGV 1-: ['C-first']
207 ARGV 1-: ['C-first']
204 ARGV 1-: ['C-second']
208 ARGV 1-: ['C-second']
205 tclass.py: deleting object: C-first
209 tclass.py: deleting object: C-first
206 ARGV 1-: ['C-third']
210 ARGV 1-: ['C-third']
207 tclass.py: deleting object: C-second
211 tclass.py: deleting object: C-second
208 tclass.py: deleting object: C-third
212 tclass.py: deleting object: C-third
209 """
213 """
210 tt.ipexec_validate(self.fname, out)
214 if dec.module_not_available('sqlite3'):
215 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
216 else:
217 err = None
218 tt.ipexec_validate(self.fname, out, err)
@@ -1,400 +1,408 b''
1 """A TaskRecord backend using sqlite3
1 """A TaskRecord backend using sqlite3
2
2
3 Authors:
3 Authors:
4
4
5 * Min RK
5 * Min RK
6 """
6 """
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2011 The IPython Development Team
8 # Copyright (C) 2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 import json
14 import json
15 import os
15 import os
16 import cPickle as pickle
16 import cPickle as pickle
17 from datetime import datetime
17 from datetime import datetime
18
18
19 try:
19 import sqlite3
20 import sqlite3
21 except ImportError:
22 sqlite3 = None
20
23
21 from zmq.eventloop import ioloop
24 from zmq.eventloop import ioloop
22
25
23 from IPython.utils.traitlets import Unicode, Instance, List, Dict
26 from IPython.utils.traitlets import Unicode, Instance, List, Dict
24 from .dictdb import BaseDB
27 from .dictdb import BaseDB
25 from IPython.utils.jsonutil import date_default, extract_dates, squash_dates
28 from IPython.utils.jsonutil import date_default, extract_dates, squash_dates
26
29
27 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
28 # SQLite operators, adapters, and converters
31 # SQLite operators, adapters, and converters
29 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
30
33
31 try:
34 try:
32 buffer
35 buffer
33 except NameError:
36 except NameError:
34 # py3k
37 # py3k
35 buffer = memoryview
38 buffer = memoryview
36
39
37 operators = {
40 operators = {
38 '$lt' : "<",
41 '$lt' : "<",
39 '$gt' : ">",
42 '$gt' : ">",
40 # null is handled weird with ==,!=
43 # null is handled weird with ==,!=
41 '$eq' : "=",
44 '$eq' : "=",
42 '$ne' : "!=",
45 '$ne' : "!=",
43 '$lte': "<=",
46 '$lte': "<=",
44 '$gte': ">=",
47 '$gte': ">=",
45 '$in' : ('=', ' OR '),
48 '$in' : ('=', ' OR '),
46 '$nin': ('!=', ' AND '),
49 '$nin': ('!=', ' AND '),
47 # '$all': None,
50 # '$all': None,
48 # '$mod': None,
51 # '$mod': None,
49 # '$exists' : None
52 # '$exists' : None
50 }
53 }
51 null_operators = {
54 null_operators = {
52 '=' : "IS NULL",
55 '=' : "IS NULL",
53 '!=' : "IS NOT NULL",
56 '!=' : "IS NOT NULL",
54 }
57 }
55
58
56 def _adapt_dict(d):
59 def _adapt_dict(d):
57 return json.dumps(d, default=date_default)
60 return json.dumps(d, default=date_default)
58
61
59 def _convert_dict(ds):
62 def _convert_dict(ds):
60 if ds is None:
63 if ds is None:
61 return ds
64 return ds
62 else:
65 else:
63 if isinstance(ds, bytes):
66 if isinstance(ds, bytes):
64 # If I understand the sqlite doc correctly, this will always be utf8
67 # If I understand the sqlite doc correctly, this will always be utf8
65 ds = ds.decode('utf8')
68 ds = ds.decode('utf8')
66 return extract_dates(json.loads(ds))
69 return extract_dates(json.loads(ds))
67
70
68 def _adapt_bufs(bufs):
71 def _adapt_bufs(bufs):
69 # this is *horrible*
72 # this is *horrible*
70 # copy buffers into single list and pickle it:
73 # copy buffers into single list and pickle it:
71 if bufs and isinstance(bufs[0], (bytes, buffer)):
74 if bufs and isinstance(bufs[0], (bytes, buffer)):
72 return sqlite3.Binary(pickle.dumps(map(bytes, bufs),-1))
75 return sqlite3.Binary(pickle.dumps(map(bytes, bufs),-1))
73 elif bufs:
76 elif bufs:
74 return bufs
77 return bufs
75 else:
78 else:
76 return None
79 return None
77
80
78 def _convert_bufs(bs):
81 def _convert_bufs(bs):
79 if bs is None:
82 if bs is None:
80 return []
83 return []
81 else:
84 else:
82 return pickle.loads(bytes(bs))
85 return pickle.loads(bytes(bs))
83
86
84 #-----------------------------------------------------------------------------
87 #-----------------------------------------------------------------------------
85 # SQLiteDB class
88 # SQLiteDB class
86 #-----------------------------------------------------------------------------
89 #-----------------------------------------------------------------------------
87
90
88 class SQLiteDB(BaseDB):
91 class SQLiteDB(BaseDB):
89 """SQLite3 TaskRecord backend."""
92 """SQLite3 TaskRecord backend."""
90
93
91 filename = Unicode('tasks.db', config=True,
94 filename = Unicode('tasks.db', config=True,
92 help="""The filename of the sqlite task database. [default: 'tasks.db']""")
95 help="""The filename of the sqlite task database. [default: 'tasks.db']""")
93 location = Unicode('', config=True,
96 location = Unicode('', config=True,
94 help="""The directory containing the sqlite task database. The default
97 help="""The directory containing the sqlite task database. The default
95 is to use the cluster_dir location.""")
98 is to use the cluster_dir location.""")
96 table = Unicode("", config=True,
99 table = Unicode("", config=True,
97 help="""The SQLite Table to use for storing tasks for this session. If unspecified,
100 help="""The SQLite Table to use for storing tasks for this session. If unspecified,
98 a new table will be created with the Hub's IDENT. Specifying the table will result
101 a new table will be created with the Hub's IDENT. Specifying the table will result
99 in tasks from previous sessions being available via Clients' db_query and
102 in tasks from previous sessions being available via Clients' db_query and
100 get_result methods.""")
103 get_result methods.""")
101
104
105 if sqlite3 is not None:
102 _db = Instance('sqlite3.Connection')
106 _db = Instance('sqlite3.Connection')
107 else:
108 _db = None
103 # the ordered list of column names
109 # the ordered list of column names
104 _keys = List(['msg_id' ,
110 _keys = List(['msg_id' ,
105 'header' ,
111 'header' ,
106 'content',
112 'content',
107 'buffers',
113 'buffers',
108 'submitted',
114 'submitted',
109 'client_uuid' ,
115 'client_uuid' ,
110 'engine_uuid' ,
116 'engine_uuid' ,
111 'started',
117 'started',
112 'completed',
118 'completed',
113 'resubmitted',
119 'resubmitted',
114 'result_header' ,
120 'result_header' ,
115 'result_content' ,
121 'result_content' ,
116 'result_buffers' ,
122 'result_buffers' ,
117 'queue' ,
123 'queue' ,
118 'pyin' ,
124 'pyin' ,
119 'pyout',
125 'pyout',
120 'pyerr',
126 'pyerr',
121 'stdout',
127 'stdout',
122 'stderr',
128 'stderr',
123 ])
129 ])
124 # sqlite datatypes for checking that db is current format
130 # sqlite datatypes for checking that db is current format
125 _types = Dict({'msg_id' : 'text' ,
131 _types = Dict({'msg_id' : 'text' ,
126 'header' : 'dict text',
132 'header' : 'dict text',
127 'content' : 'dict text',
133 'content' : 'dict text',
128 'buffers' : 'bufs blob',
134 'buffers' : 'bufs blob',
129 'submitted' : 'timestamp',
135 'submitted' : 'timestamp',
130 'client_uuid' : 'text',
136 'client_uuid' : 'text',
131 'engine_uuid' : 'text',
137 'engine_uuid' : 'text',
132 'started' : 'timestamp',
138 'started' : 'timestamp',
133 'completed' : 'timestamp',
139 'completed' : 'timestamp',
134 'resubmitted' : 'timestamp',
140 'resubmitted' : 'timestamp',
135 'result_header' : 'dict text',
141 'result_header' : 'dict text',
136 'result_content' : 'dict text',
142 'result_content' : 'dict text',
137 'result_buffers' : 'bufs blob',
143 'result_buffers' : 'bufs blob',
138 'queue' : 'text',
144 'queue' : 'text',
139 'pyin' : 'text',
145 'pyin' : 'text',
140 'pyout' : 'text',
146 'pyout' : 'text',
141 'pyerr' : 'text',
147 'pyerr' : 'text',
142 'stdout' : 'text',
148 'stdout' : 'text',
143 'stderr' : 'text',
149 'stderr' : 'text',
144 })
150 })
145
151
146 def __init__(self, **kwargs):
152 def __init__(self, **kwargs):
147 super(SQLiteDB, self).__init__(**kwargs)
153 super(SQLiteDB, self).__init__(**kwargs)
154 if sqlite3 is None:
155 raise ImportError("SQLiteDB requires sqlite3")
148 if not self.table:
156 if not self.table:
149 # use session, and prefix _, since starting with # is illegal
157 # use session, and prefix _, since starting with # is illegal
150 self.table = '_'+self.session.replace('-','_')
158 self.table = '_'+self.session.replace('-','_')
151 if not self.location:
159 if not self.location:
152 # get current profile
160 # get current profile
153 from IPython.core.application import BaseIPythonApplication
161 from IPython.core.application import BaseIPythonApplication
154 if BaseIPythonApplication.initialized():
162 if BaseIPythonApplication.initialized():
155 app = BaseIPythonApplication.instance()
163 app = BaseIPythonApplication.instance()
156 if app.profile_dir is not None:
164 if app.profile_dir is not None:
157 self.location = app.profile_dir.location
165 self.location = app.profile_dir.location
158 else:
166 else:
159 self.location = u'.'
167 self.location = u'.'
160 else:
168 else:
161 self.location = u'.'
169 self.location = u'.'
162 self._init_db()
170 self._init_db()
163
171
164 # register db commit as 2s periodic callback
172 # register db commit as 2s periodic callback
165 # to prevent clogging pipes
173 # to prevent clogging pipes
166 # assumes we are being run in a zmq ioloop app
174 # assumes we are being run in a zmq ioloop app
167 loop = ioloop.IOLoop.instance()
175 loop = ioloop.IOLoop.instance()
168 pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
176 pc = ioloop.PeriodicCallback(self._db.commit, 2000, loop)
169 pc.start()
177 pc.start()
170
178
171 def _defaults(self, keys=None):
179 def _defaults(self, keys=None):
172 """create an empty record"""
180 """create an empty record"""
173 d = {}
181 d = {}
174 keys = self._keys if keys is None else keys
182 keys = self._keys if keys is None else keys
175 for key in keys:
183 for key in keys:
176 d[key] = None
184 d[key] = None
177 return d
185 return d
178
186
179 def _check_table(self):
187 def _check_table(self):
180 """Ensure that an incorrect table doesn't exist
188 """Ensure that an incorrect table doesn't exist
181
189
182 If a bad (old) table does exist, return False
190 If a bad (old) table does exist, return False
183 """
191 """
184 cursor = self._db.execute("PRAGMA table_info(%s)"%self.table)
192 cursor = self._db.execute("PRAGMA table_info(%s)"%self.table)
185 lines = cursor.fetchall()
193 lines = cursor.fetchall()
186 if not lines:
194 if not lines:
187 # table does not exist
195 # table does not exist
188 return True
196 return True
189 types = {}
197 types = {}
190 keys = []
198 keys = []
191 for line in lines:
199 for line in lines:
192 keys.append(line[1])
200 keys.append(line[1])
193 types[line[1]] = line[2]
201 types[line[1]] = line[2]
194 if self._keys != keys:
202 if self._keys != keys:
195 # key mismatch
203 # key mismatch
196 self.log.warn('keys mismatch')
204 self.log.warn('keys mismatch')
197 return False
205 return False
198 for key in self._keys:
206 for key in self._keys:
199 if types[key] != self._types[key]:
207 if types[key] != self._types[key]:
200 self.log.warn(
208 self.log.warn(
201 'type mismatch: %s: %s != %s'%(key,types[key],self._types[key])
209 'type mismatch: %s: %s != %s'%(key,types[key],self._types[key])
202 )
210 )
203 return False
211 return False
204 return True
212 return True
205
213
206 def _init_db(self):
214 def _init_db(self):
207 """Connect to the database and get new session number."""
215 """Connect to the database and get new session number."""
208 # register adapters
216 # register adapters
209 sqlite3.register_adapter(dict, _adapt_dict)
217 sqlite3.register_adapter(dict, _adapt_dict)
210 sqlite3.register_converter('dict', _convert_dict)
218 sqlite3.register_converter('dict', _convert_dict)
211 sqlite3.register_adapter(list, _adapt_bufs)
219 sqlite3.register_adapter(list, _adapt_bufs)
212 sqlite3.register_converter('bufs', _convert_bufs)
220 sqlite3.register_converter('bufs', _convert_bufs)
213 # connect to the db
221 # connect to the db
214 dbfile = os.path.join(self.location, self.filename)
222 dbfile = os.path.join(self.location, self.filename)
215 self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES,
223 self._db = sqlite3.connect(dbfile, detect_types=sqlite3.PARSE_DECLTYPES,
216 # isolation_level = None)#,
224 # isolation_level = None)#,
217 cached_statements=64)
225 cached_statements=64)
218 # print dir(self._db)
226 # print dir(self._db)
219 first_table = self.table
227 first_table = self.table
220 i=0
228 i=0
221 while not self._check_table():
229 while not self._check_table():
222 i+=1
230 i+=1
223 self.table = first_table+'_%i'%i
231 self.table = first_table+'_%i'%i
224 self.log.warn(
232 self.log.warn(
225 "Table %s exists and doesn't match db format, trying %s"%
233 "Table %s exists and doesn't match db format, trying %s"%
226 (first_table,self.table)
234 (first_table,self.table)
227 )
235 )
228
236
229 self._db.execute("""CREATE TABLE IF NOT EXISTS %s
237 self._db.execute("""CREATE TABLE IF NOT EXISTS %s
230 (msg_id text PRIMARY KEY,
238 (msg_id text PRIMARY KEY,
231 header dict text,
239 header dict text,
232 content dict text,
240 content dict text,
233 buffers bufs blob,
241 buffers bufs blob,
234 submitted timestamp,
242 submitted timestamp,
235 client_uuid text,
243 client_uuid text,
236 engine_uuid text,
244 engine_uuid text,
237 started timestamp,
245 started timestamp,
238 completed timestamp,
246 completed timestamp,
239 resubmitted timestamp,
247 resubmitted timestamp,
240 result_header dict text,
248 result_header dict text,
241 result_content dict text,
249 result_content dict text,
242 result_buffers bufs blob,
250 result_buffers bufs blob,
243 queue text,
251 queue text,
244 pyin text,
252 pyin text,
245 pyout text,
253 pyout text,
246 pyerr text,
254 pyerr text,
247 stdout text,
255 stdout text,
248 stderr text)
256 stderr text)
249 """%self.table)
257 """%self.table)
250 self._db.commit()
258 self._db.commit()
251
259
252 def _dict_to_list(self, d):
260 def _dict_to_list(self, d):
253 """turn a mongodb-style record dict into a list."""
261 """turn a mongodb-style record dict into a list."""
254
262
255 return [ d[key] for key in self._keys ]
263 return [ d[key] for key in self._keys ]
256
264
257 def _list_to_dict(self, line, keys=None):
265 def _list_to_dict(self, line, keys=None):
258 """Inverse of dict_to_list"""
266 """Inverse of dict_to_list"""
259 keys = self._keys if keys is None else keys
267 keys = self._keys if keys is None else keys
260 d = self._defaults(keys)
268 d = self._defaults(keys)
261 for key,value in zip(keys, line):
269 for key,value in zip(keys, line):
262 d[key] = value
270 d[key] = value
263
271
264 return d
272 return d
265
273
266 def _render_expression(self, check):
274 def _render_expression(self, check):
267 """Turn a mongodb-style search dict into an SQL query."""
275 """Turn a mongodb-style search dict into an SQL query."""
268 expressions = []
276 expressions = []
269 args = []
277 args = []
270
278
271 skeys = set(check.keys())
279 skeys = set(check.keys())
272 skeys.difference_update(set(self._keys))
280 skeys.difference_update(set(self._keys))
273 skeys.difference_update(set(['buffers', 'result_buffers']))
281 skeys.difference_update(set(['buffers', 'result_buffers']))
274 if skeys:
282 if skeys:
275 raise KeyError("Illegal testing key(s): %s"%skeys)
283 raise KeyError("Illegal testing key(s): %s"%skeys)
276
284
277 for name,sub_check in check.iteritems():
285 for name,sub_check in check.iteritems():
278 if isinstance(sub_check, dict):
286 if isinstance(sub_check, dict):
279 for test,value in sub_check.iteritems():
287 for test,value in sub_check.iteritems():
280 try:
288 try:
281 op = operators[test]
289 op = operators[test]
282 except KeyError:
290 except KeyError:
283 raise KeyError("Unsupported operator: %r"%test)
291 raise KeyError("Unsupported operator: %r"%test)
284 if isinstance(op, tuple):
292 if isinstance(op, tuple):
285 op, join = op
293 op, join = op
286
294
287 if value is None and op in null_operators:
295 if value is None and op in null_operators:
288 expr = "%s %s"%null_operators[op]
296 expr = "%s %s"%null_operators[op]
289 else:
297 else:
290 expr = "%s %s ?"%(name, op)
298 expr = "%s %s ?"%(name, op)
291 if isinstance(value, (tuple,list)):
299 if isinstance(value, (tuple,list)):
292 if op in null_operators and any([v is None for v in value]):
300 if op in null_operators and any([v is None for v in value]):
293 # equality tests don't work with NULL
301 # equality tests don't work with NULL
294 raise ValueError("Cannot use %r test with NULL values on SQLite backend"%test)
302 raise ValueError("Cannot use %r test with NULL values on SQLite backend"%test)
295 expr = '( %s )'%( join.join([expr]*len(value)) )
303 expr = '( %s )'%( join.join([expr]*len(value)) )
296 args.extend(value)
304 args.extend(value)
297 else:
305 else:
298 args.append(value)
306 args.append(value)
299 expressions.append(expr)
307 expressions.append(expr)
300 else:
308 else:
301 # it's an equality check
309 # it's an equality check
302 if sub_check is None:
310 if sub_check is None:
303 expressions.append("%s IS NULL")
311 expressions.append("%s IS NULL")
304 else:
312 else:
305 expressions.append("%s = ?"%name)
313 expressions.append("%s = ?"%name)
306 args.append(sub_check)
314 args.append(sub_check)
307
315
308 expr = " AND ".join(expressions)
316 expr = " AND ".join(expressions)
309 return expr, args
317 return expr, args
310
318
311 def add_record(self, msg_id, rec):
319 def add_record(self, msg_id, rec):
312 """Add a new Task Record, by msg_id."""
320 """Add a new Task Record, by msg_id."""
313 d = self._defaults()
321 d = self._defaults()
314 d.update(rec)
322 d.update(rec)
315 d['msg_id'] = msg_id
323 d['msg_id'] = msg_id
316 line = self._dict_to_list(d)
324 line = self._dict_to_list(d)
317 tups = '(%s)'%(','.join(['?']*len(line)))
325 tups = '(%s)'%(','.join(['?']*len(line)))
318 self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
326 self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
319 # self._db.commit()
327 # self._db.commit()
320
328
321 def get_record(self, msg_id):
329 def get_record(self, msg_id):
322 """Get a specific Task Record, by msg_id."""
330 """Get a specific Task Record, by msg_id."""
323 cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
331 cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
324 line = cursor.fetchone()
332 line = cursor.fetchone()
325 if line is None:
333 if line is None:
326 raise KeyError("No such msg: %r"%msg_id)
334 raise KeyError("No such msg: %r"%msg_id)
327 return self._list_to_dict(line)
335 return self._list_to_dict(line)
328
336
329 def update_record(self, msg_id, rec):
337 def update_record(self, msg_id, rec):
330 """Update the data in an existing record."""
338 """Update the data in an existing record."""
331 query = "UPDATE %s SET "%self.table
339 query = "UPDATE %s SET "%self.table
332 sets = []
340 sets = []
333 keys = sorted(rec.keys())
341 keys = sorted(rec.keys())
334 values = []
342 values = []
335 for key in keys:
343 for key in keys:
336 sets.append('%s = ?'%key)
344 sets.append('%s = ?'%key)
337 values.append(rec[key])
345 values.append(rec[key])
338 query += ', '.join(sets)
346 query += ', '.join(sets)
339 query += ' WHERE msg_id == ?'
347 query += ' WHERE msg_id == ?'
340 values.append(msg_id)
348 values.append(msg_id)
341 self._db.execute(query, values)
349 self._db.execute(query, values)
342 # self._db.commit()
350 # self._db.commit()
343
351
344 def drop_record(self, msg_id):
352 def drop_record(self, msg_id):
345 """Remove a record from the DB."""
353 """Remove a record from the DB."""
346 self._db.execute("""DELETE FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
354 self._db.execute("""DELETE FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
347 # self._db.commit()
355 # self._db.commit()
348
356
349 def drop_matching_records(self, check):
357 def drop_matching_records(self, check):
350 """Remove a record from the DB."""
358 """Remove a record from the DB."""
351 expr,args = self._render_expression(check)
359 expr,args = self._render_expression(check)
352 query = "DELETE FROM %s WHERE %s"%(self.table, expr)
360 query = "DELETE FROM %s WHERE %s"%(self.table, expr)
353 self._db.execute(query,args)
361 self._db.execute(query,args)
354 # self._db.commit()
362 # self._db.commit()
355
363
356 def find_records(self, check, keys=None):
364 def find_records(self, check, keys=None):
357 """Find records matching a query dict, optionally extracting subset of keys.
365 """Find records matching a query dict, optionally extracting subset of keys.
358
366
359 Returns list of matching records.
367 Returns list of matching records.
360
368
361 Parameters
369 Parameters
362 ----------
370 ----------
363
371
364 check: dict
372 check: dict
365 mongodb-style query argument
373 mongodb-style query argument
366 keys: list of strs [optional]
374 keys: list of strs [optional]
367 if specified, the subset of keys to extract. msg_id will *always* be
375 if specified, the subset of keys to extract. msg_id will *always* be
368 included.
376 included.
369 """
377 """
370 if keys:
378 if keys:
371 bad_keys = [ key for key in keys if key not in self._keys ]
379 bad_keys = [ key for key in keys if key not in self._keys ]
372 if bad_keys:
380 if bad_keys:
373 raise KeyError("Bad record key(s): %s"%bad_keys)
381 raise KeyError("Bad record key(s): %s"%bad_keys)
374
382
375 if keys:
383 if keys:
376 # ensure msg_id is present and first:
384 # ensure msg_id is present and first:
377 if 'msg_id' in keys:
385 if 'msg_id' in keys:
378 keys.remove('msg_id')
386 keys.remove('msg_id')
379 keys.insert(0, 'msg_id')
387 keys.insert(0, 'msg_id')
380 req = ', '.join(keys)
388 req = ', '.join(keys)
381 else:
389 else:
382 req = '*'
390 req = '*'
383 expr,args = self._render_expression(check)
391 expr,args = self._render_expression(check)
384 query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
392 query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
385 cursor = self._db.execute(query, args)
393 cursor = self._db.execute(query, args)
386 matches = cursor.fetchall()
394 matches = cursor.fetchall()
387 records = []
395 records = []
388 for line in matches:
396 for line in matches:
389 rec = self._list_to_dict(line, keys)
397 rec = self._list_to_dict(line, keys)
390 records.append(rec)
398 records.append(rec)
391 return records
399 return records
392
400
393 def get_history(self):
401 def get_history(self):
394 """get all msg_ids, ordered by time submitted."""
402 """get all msg_ids, ordered by time submitted."""
395 query = """SELECT msg_id FROM %s ORDER by submitted ASC"""%self.table
403 query = """SELECT msg_id FROM %s ORDER by submitted ASC"""%self.table
396 cursor = self._db.execute(query)
404 cursor = self._db.execute(query)
397 # will be a list of length 1 tuples
405 # will be a list of length 1 tuples
398 return [ tup[0] for tup in cursor.fetchall()]
406 return [ tup[0] for tup in cursor.fetchall()]
399
407
400 __all__ = ['SQLiteDB'] No newline at end of file
408 __all__ = ['SQLiteDB']
@@ -1,180 +1,182 b''
1 """Tests for db backends
1 """Tests for db backends
2
2
3 Authors:
3 Authors:
4
4
5 * Min RK
5 * Min RK
6 """
6 """
7
7
8 #-------------------------------------------------------------------------------
8 #-------------------------------------------------------------------------------
9 # Copyright (C) 2011 The IPython Development Team
9 # Copyright (C) 2011 The IPython Development Team
10 #
10 #
11 # Distributed under the terms of the BSD License. The full license is in
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
12 # the file COPYING, distributed as part of this software.
13 #-------------------------------------------------------------------------------
13 #-------------------------------------------------------------------------------
14
14
15 #-------------------------------------------------------------------------------
15 #-------------------------------------------------------------------------------
16 # Imports
16 # Imports
17 #-------------------------------------------------------------------------------
17 #-------------------------------------------------------------------------------
18
18
19 from __future__ import division
19 from __future__ import division
20
20
21 import tempfile
21 import tempfile
22 import time
22 import time
23
23
24 from datetime import datetime, timedelta
24 from datetime import datetime, timedelta
25 from unittest import TestCase
25 from unittest import TestCase
26
26
27 from nose import SkipTest
28
29 from IPython.parallel import error
27 from IPython.parallel import error
30 from IPython.parallel.controller.dictdb import DictDB
28 from IPython.parallel.controller.dictdb import DictDB
31 from IPython.parallel.controller.sqlitedb import SQLiteDB
29 from IPython.parallel.controller.sqlitedb import SQLiteDB
32 from IPython.parallel.controller.hub import init_record, empty_record
30 from IPython.parallel.controller.hub import init_record, empty_record
33
31
32 from IPython.testing import decorators as dec
34 from IPython.zmq.session import Session
33 from IPython.zmq.session import Session
35
34
36
35
37 #-------------------------------------------------------------------------------
36 #-------------------------------------------------------------------------------
38 # TestCases
37 # TestCases
39 #-------------------------------------------------------------------------------
38 #-------------------------------------------------------------------------------
40
39
41 class TestDictBackend(TestCase):
40 class TestDictBackend(TestCase):
42 def setUp(self):
41 def setUp(self):
43 self.session = Session()
42 self.session = Session()
44 self.db = self.create_db()
43 self.db = self.create_db()
45 self.load_records(16)
44 self.load_records(16)
46
45
47 def create_db(self):
46 def create_db(self):
48 return DictDB()
47 return DictDB()
49
48
50 def load_records(self, n=1):
49 def load_records(self, n=1):
51 """load n records for testing"""
50 """load n records for testing"""
52 #sleep 1/10 s, to ensure timestamp is different to previous calls
51 #sleep 1/10 s, to ensure timestamp is different to previous calls
53 time.sleep(0.1)
52 time.sleep(0.1)
54 msg_ids = []
53 msg_ids = []
55 for i in range(n):
54 for i in range(n):
56 msg = self.session.msg('apply_request', content=dict(a=5))
55 msg = self.session.msg('apply_request', content=dict(a=5))
57 msg['buffers'] = []
56 msg['buffers'] = []
58 rec = init_record(msg)
57 rec = init_record(msg)
59 msg_id = msg['header']['msg_id']
58 msg_id = msg['header']['msg_id']
60 msg_ids.append(msg_id)
59 msg_ids.append(msg_id)
61 self.db.add_record(msg_id, rec)
60 self.db.add_record(msg_id, rec)
62 return msg_ids
61 return msg_ids
63
62
64 def test_add_record(self):
63 def test_add_record(self):
65 before = self.db.get_history()
64 before = self.db.get_history()
66 self.load_records(5)
65 self.load_records(5)
67 after = self.db.get_history()
66 after = self.db.get_history()
68 self.assertEquals(len(after), len(before)+5)
67 self.assertEquals(len(after), len(before)+5)
69 self.assertEquals(after[:-5],before)
68 self.assertEquals(after[:-5],before)
70
69
71 def test_drop_record(self):
70 def test_drop_record(self):
72 msg_id = self.load_records()[-1]
71 msg_id = self.load_records()[-1]
73 rec = self.db.get_record(msg_id)
72 rec = self.db.get_record(msg_id)
74 self.db.drop_record(msg_id)
73 self.db.drop_record(msg_id)
75 self.assertRaises(KeyError,self.db.get_record, msg_id)
74 self.assertRaises(KeyError,self.db.get_record, msg_id)
76
75
77 def _round_to_millisecond(self, dt):
76 def _round_to_millisecond(self, dt):
78 """necessary because mongodb rounds microseconds"""
77 """necessary because mongodb rounds microseconds"""
79 micro = dt.microsecond
78 micro = dt.microsecond
80 extra = int(str(micro)[-3:])
79 extra = int(str(micro)[-3:])
81 return dt - timedelta(microseconds=extra)
80 return dt - timedelta(microseconds=extra)
82
81
83 def test_update_record(self):
82 def test_update_record(self):
84 now = self._round_to_millisecond(datetime.now())
83 now = self._round_to_millisecond(datetime.now())
85 #
84 #
86 msg_id = self.db.get_history()[-1]
85 msg_id = self.db.get_history()[-1]
87 rec1 = self.db.get_record(msg_id)
86 rec1 = self.db.get_record(msg_id)
88 data = {'stdout': 'hello there', 'completed' : now}
87 data = {'stdout': 'hello there', 'completed' : now}
89 self.db.update_record(msg_id, data)
88 self.db.update_record(msg_id, data)
90 rec2 = self.db.get_record(msg_id)
89 rec2 = self.db.get_record(msg_id)
91 self.assertEquals(rec2['stdout'], 'hello there')
90 self.assertEquals(rec2['stdout'], 'hello there')
92 self.assertEquals(rec2['completed'], now)
91 self.assertEquals(rec2['completed'], now)
93 rec1.update(data)
92 rec1.update(data)
94 self.assertEquals(rec1, rec2)
93 self.assertEquals(rec1, rec2)
95
94
96 # def test_update_record_bad(self):
95 # def test_update_record_bad(self):
97 # """test updating nonexistant records"""
96 # """test updating nonexistant records"""
98 # msg_id = str(uuid.uuid4())
97 # msg_id = str(uuid.uuid4())
99 # data = {'stdout': 'hello there'}
98 # data = {'stdout': 'hello there'}
100 # self.assertRaises(KeyError, self.db.update_record, msg_id, data)
99 # self.assertRaises(KeyError, self.db.update_record, msg_id, data)
101
100
102 def test_find_records_dt(self):
101 def test_find_records_dt(self):
103 """test finding records by date"""
102 """test finding records by date"""
104 hist = self.db.get_history()
103 hist = self.db.get_history()
105 middle = self.db.get_record(hist[len(hist)//2])
104 middle = self.db.get_record(hist[len(hist)//2])
106 tic = middle['submitted']
105 tic = middle['submitted']
107 before = self.db.find_records({'submitted' : {'$lt' : tic}})
106 before = self.db.find_records({'submitted' : {'$lt' : tic}})
108 after = self.db.find_records({'submitted' : {'$gte' : tic}})
107 after = self.db.find_records({'submitted' : {'$gte' : tic}})
109 self.assertEquals(len(before)+len(after),len(hist))
108 self.assertEquals(len(before)+len(after),len(hist))
110 for b in before:
109 for b in before:
111 self.assertTrue(b['submitted'] < tic)
110 self.assertTrue(b['submitted'] < tic)
112 for a in after:
111 for a in after:
113 self.assertTrue(a['submitted'] >= tic)
112 self.assertTrue(a['submitted'] >= tic)
114 same = self.db.find_records({'submitted' : tic})
113 same = self.db.find_records({'submitted' : tic})
115 for s in same:
114 for s in same:
116 self.assertTrue(s['submitted'] == tic)
115 self.assertTrue(s['submitted'] == tic)
117
116
118 def test_find_records_keys(self):
117 def test_find_records_keys(self):
119 """test extracting subset of record keys"""
118 """test extracting subset of record keys"""
120 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
119 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
121 for rec in found:
120 for rec in found:
122 self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
121 self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))
123
122
124 def test_find_records_msg_id(self):
123 def test_find_records_msg_id(self):
125 """ensure msg_id is always in found records"""
124 """ensure msg_id is always in found records"""
126 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
125 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
127 for rec in found:
126 for rec in found:
128 self.assertTrue('msg_id' in rec.keys())
127 self.assertTrue('msg_id' in rec.keys())
129 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted'])
128 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['submitted'])
130 for rec in found:
129 for rec in found:
131 self.assertTrue('msg_id' in rec.keys())
130 self.assertTrue('msg_id' in rec.keys())
132 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['msg_id'])
131 found = self.db.find_records({'msg_id': {'$ne' : ''}},keys=['msg_id'])
133 for rec in found:
132 for rec in found:
134 self.assertTrue('msg_id' in rec.keys())
133 self.assertTrue('msg_id' in rec.keys())
135
134
136 def test_find_records_in(self):
135 def test_find_records_in(self):
137 """test finding records with '$in','$nin' operators"""
136 """test finding records with '$in','$nin' operators"""
138 hist = self.db.get_history()
137 hist = self.db.get_history()
139 even = hist[::2]
138 even = hist[::2]
140 odd = hist[1::2]
139 odd = hist[1::2]
141 recs = self.db.find_records({ 'msg_id' : {'$in' : even}})
140 recs = self.db.find_records({ 'msg_id' : {'$in' : even}})
142 found = [ r['msg_id'] for r in recs ]
141 found = [ r['msg_id'] for r in recs ]
143 self.assertEquals(set(even), set(found))
142 self.assertEquals(set(even), set(found))
144 recs = self.db.find_records({ 'msg_id' : {'$nin' : even}})
143 recs = self.db.find_records({ 'msg_id' : {'$nin' : even}})
145 found = [ r['msg_id'] for r in recs ]
144 found = [ r['msg_id'] for r in recs ]
146 self.assertEquals(set(odd), set(found))
145 self.assertEquals(set(odd), set(found))
147
146
148 def test_get_history(self):
147 def test_get_history(self):
149 msg_ids = self.db.get_history()
148 msg_ids = self.db.get_history()
150 latest = datetime(1984,1,1)
149 latest = datetime(1984,1,1)
151 for msg_id in msg_ids:
150 for msg_id in msg_ids:
152 rec = self.db.get_record(msg_id)
151 rec = self.db.get_record(msg_id)
153 newt = rec['submitted']
152 newt = rec['submitted']
154 self.assertTrue(newt >= latest)
153 self.assertTrue(newt >= latest)
155 latest = newt
154 latest = newt
156 msg_id = self.load_records(1)[-1]
155 msg_id = self.load_records(1)[-1]
157 self.assertEquals(self.db.get_history()[-1],msg_id)
156 self.assertEquals(self.db.get_history()[-1],msg_id)
158
157
159 def test_datetime(self):
158 def test_datetime(self):
160 """get/set timestamps with datetime objects"""
159 """get/set timestamps with datetime objects"""
161 msg_id = self.db.get_history()[-1]
160 msg_id = self.db.get_history()[-1]
162 rec = self.db.get_record(msg_id)
161 rec = self.db.get_record(msg_id)
163 self.assertTrue(isinstance(rec['submitted'], datetime))
162 self.assertTrue(isinstance(rec['submitted'], datetime))
164 self.db.update_record(msg_id, dict(completed=datetime.now()))
163 self.db.update_record(msg_id, dict(completed=datetime.now()))
165 rec = self.db.get_record(msg_id)
164 rec = self.db.get_record(msg_id)
166 self.assertTrue(isinstance(rec['completed'], datetime))
165 self.assertTrue(isinstance(rec['completed'], datetime))
167
166
168 def test_drop_matching(self):
167 def test_drop_matching(self):
169 msg_ids = self.load_records(10)
168 msg_ids = self.load_records(10)
170 query = {'msg_id' : {'$in':msg_ids}}
169 query = {'msg_id' : {'$in':msg_ids}}
171 self.db.drop_matching_records(query)
170 self.db.drop_matching_records(query)
172 recs = self.db.find_records(query)
171 recs = self.db.find_records(query)
173 self.assertEquals(len(recs), 0)
172 self.assertEquals(len(recs), 0)
174
173
174
175 class TestSQLiteBackend(TestDictBackend):
175 class TestSQLiteBackend(TestDictBackend):
176
177 @dec.skip_without('sqlite3')
176 def create_db(self):
178 def create_db(self):
177 return SQLiteDB(location=tempfile.gettempdir())
179 return SQLiteDB(location=tempfile.gettempdir())
178
180
179 def tearDown(self):
181 def tearDown(self):
180 self.db._db.close()
182 self.db._db.close()
@@ -1,473 +1,476 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """IPython Test Suite Runner.
2 """IPython Test Suite Runner.
3
3
4 This module provides a main entry point to a user script to test IPython
4 This module provides a main entry point to a user script to test IPython
5 itself from the command line. There are two ways of running this script:
5 itself from the command line. There are two ways of running this script:
6
6
7 1. With the syntax `iptest all`. This runs our entire test suite by
7 1. With the syntax `iptest all`. This runs our entire test suite by
8 calling this script (with different arguments) recursively. This
8 calling this script (with different arguments) recursively. This
9 causes modules and package to be tested in different processes, using nose
9 causes modules and package to be tested in different processes, using nose
10 or trial where appropriate.
10 or trial where appropriate.
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 the script simply calls nose, but with special command line flags and
12 the script simply calls nose, but with special command line flags and
13 plugins loaded.
13 plugins loaded.
14
14
15 """
15 """
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2009 The IPython Development Team
18 # Copyright (C) 2009 The IPython Development Team
19 #
19 #
20 # Distributed under the terms of the BSD License. The full license is in
20 # Distributed under the terms of the BSD License. The full license is in
21 # the file COPYING, distributed as part of this software.
21 # the file COPYING, distributed as part of this software.
22 #-----------------------------------------------------------------------------
22 #-----------------------------------------------------------------------------
23
23
24 #-----------------------------------------------------------------------------
24 #-----------------------------------------------------------------------------
25 # Imports
25 # Imports
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27
27
28 # Stdlib
28 # Stdlib
29 import os
29 import os
30 import os.path as path
30 import os.path as path
31 import signal
31 import signal
32 import sys
32 import sys
33 import subprocess
33 import subprocess
34 import tempfile
34 import tempfile
35 import time
35 import time
36 import warnings
36 import warnings
37
37
38 # Note: monkeypatch!
38 # Note: monkeypatch!
39 # We need to monkeypatch a small problem in nose itself first, before importing
39 # We need to monkeypatch a small problem in nose itself first, before importing
40 # it for actual use. This should get into nose upstream, but its release cycle
40 # it for actual use. This should get into nose upstream, but its release cycle
41 # is slow and we need it for our parametric tests to work correctly.
41 # is slow and we need it for our parametric tests to work correctly.
42 from IPython.testing import nosepatch
42 from IPython.testing import nosepatch
43 # Now, proceed to import nose itself
43 # Now, proceed to import nose itself
44 import nose.plugins.builtin
44 import nose.plugins.builtin
45 from nose.core import TestProgram
45 from nose.core import TestProgram
46
46
47 # Our own imports
47 # Our own imports
48 from IPython.utils.importstring import import_item
48 from IPython.utils.importstring import import_item
49 from IPython.utils.path import get_ipython_module_path
49 from IPython.utils.path import get_ipython_module_path
50 from IPython.utils.process import find_cmd, pycmd2argv
50 from IPython.utils.process import find_cmd, pycmd2argv
51 from IPython.utils.sysinfo import sys_info
51 from IPython.utils.sysinfo import sys_info
52
52
53 from IPython.testing import globalipapp
53 from IPython.testing import globalipapp
54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
54 from IPython.testing.plugin.ipdoctest import IPythonDoctest
55 from IPython.external.decorators import KnownFailure
55 from IPython.external.decorators import KnownFailure
56
56
57 pjoin = path.join
57 pjoin = path.join
58
58
59
59
60 #-----------------------------------------------------------------------------
60 #-----------------------------------------------------------------------------
61 # Globals
61 # Globals
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63
63
64
64
65 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
66 # Warnings control
66 # Warnings control
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68
68
69 # Twisted generates annoying warnings with Python 2.6, as will do other code
69 # Twisted generates annoying warnings with Python 2.6, as will do other code
70 # that imports 'sets' as of today
70 # that imports 'sets' as of today
71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
71 warnings.filterwarnings('ignore', 'the sets module is deprecated',
72 DeprecationWarning )
72 DeprecationWarning )
73
73
74 # This one also comes from Twisted
74 # This one also comes from Twisted
75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
75 warnings.filterwarnings('ignore', 'the sha module is deprecated',
76 DeprecationWarning)
76 DeprecationWarning)
77
77
78 # Wx on Fedora11 spits these out
78 # Wx on Fedora11 spits these out
79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
79 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
80 UserWarning)
80 UserWarning)
81
81
82 #-----------------------------------------------------------------------------
82 #-----------------------------------------------------------------------------
83 # Logic for skipping doctests
83 # Logic for skipping doctests
84 #-----------------------------------------------------------------------------
84 #-----------------------------------------------------------------------------
85 def extract_version(mod):
85 def extract_version(mod):
86 return mod.__version__
86 return mod.__version__
87
87
88 def test_for(item, min_version=None, callback=extract_version):
88 def test_for(item, min_version=None, callback=extract_version):
89 """Test to see if item is importable, and optionally check against a minimum
89 """Test to see if item is importable, and optionally check against a minimum
90 version.
90 version.
91
91
92 If min_version is given, the default behavior is to check against the
92 If min_version is given, the default behavior is to check against the
93 `__version__` attribute of the item, but specifying `callback` allows you to
93 `__version__` attribute of the item, but specifying `callback` allows you to
94 extract the value you are interested in. e.g::
94 extract the value you are interested in. e.g::
95
95
96 In [1]: import sys
96 In [1]: import sys
97
97
98 In [2]: from IPython.testing.iptest import test_for
98 In [2]: from IPython.testing.iptest import test_for
99
99
100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
100 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
101 Out[3]: True
101 Out[3]: True
102
102
103 """
103 """
104 try:
104 try:
105 check = import_item(item)
105 check = import_item(item)
106 except (ImportError, RuntimeError):
106 except (ImportError, RuntimeError):
107 # GTK reports Runtime error if it can't be initialized even if it's
107 # GTK reports Runtime error if it can't be initialized even if it's
108 # importable.
108 # importable.
109 return False
109 return False
110 else:
110 else:
111 if min_version:
111 if min_version:
112 if callback:
112 if callback:
113 # extra processing step to get version to compare
113 # extra processing step to get version to compare
114 check = callback(check)
114 check = callback(check)
115
115
116 return check >= min_version
116 return check >= min_version
117 else:
117 else:
118 return True
118 return True
119
119
120 # Global dict where we can store information on what we have and what we don't
120 # Global dict where we can store information on what we have and what we don't
121 # have available at test run time
121 # have available at test run time
122 have = {}
122 have = {}
123
123
124 have['curses'] = test_for('_curses')
124 have['curses'] = test_for('_curses')
125 have['matplotlib'] = test_for('matplotlib')
125 have['matplotlib'] = test_for('matplotlib')
126 have['pexpect'] = test_for('IPython.external.pexpect')
126 have['pexpect'] = test_for('IPython.external.pexpect')
127 have['pymongo'] = test_for('pymongo')
127 have['pymongo'] = test_for('pymongo')
128 have['wx'] = test_for('wx')
128 have['wx'] = test_for('wx')
129 have['wx.aui'] = test_for('wx.aui')
129 have['wx.aui'] = test_for('wx.aui')
130 have['qt'] = test_for('IPython.external.qt')
130 have['qt'] = test_for('IPython.external.qt')
131 have['sqlite3'] = test_for('sqlite3')
131
132
132 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
133 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
133
134
134 if os.name == 'nt':
135 if os.name == 'nt':
135 min_zmq = (2,1,7)
136 min_zmq = (2,1,7)
136 else:
137 else:
137 min_zmq = (2,1,4)
138 min_zmq = (2,1,4)
138
139
139 def version_tuple(mod):
140 def version_tuple(mod):
140 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
141 "turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)"
141 # turn 'dev' into 999, because Python3 rejects str-int comparisons
142 # turn 'dev' into 999, because Python3 rejects str-int comparisons
142 vs = mod.__version__.replace('dev', '.999')
143 vs = mod.__version__.replace('dev', '.999')
143 tup = tuple([int(v) for v in vs.split('.') ])
144 tup = tuple([int(v) for v in vs.split('.') ])
144 return tup
145 return tup
145
146
146 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
147 have['zmq'] = test_for('zmq', min_zmq, version_tuple)
147
148
148 #-----------------------------------------------------------------------------
149 #-----------------------------------------------------------------------------
149 # Functions and classes
150 # Functions and classes
150 #-----------------------------------------------------------------------------
151 #-----------------------------------------------------------------------------
151
152
152 def report():
153 def report():
153 """Return a string with a summary report of test-related variables."""
154 """Return a string with a summary report of test-related variables."""
154
155
155 out = [ sys_info(), '\n']
156 out = [ sys_info(), '\n']
156
157
157 avail = []
158 avail = []
158 not_avail = []
159 not_avail = []
159
160
160 for k, is_avail in have.items():
161 for k, is_avail in have.items():
161 if is_avail:
162 if is_avail:
162 avail.append(k)
163 avail.append(k)
163 else:
164 else:
164 not_avail.append(k)
165 not_avail.append(k)
165
166
166 if avail:
167 if avail:
167 out.append('\nTools and libraries available at test time:\n')
168 out.append('\nTools and libraries available at test time:\n')
168 avail.sort()
169 avail.sort()
169 out.append(' ' + ' '.join(avail)+'\n')
170 out.append(' ' + ' '.join(avail)+'\n')
170
171
171 if not_avail:
172 if not_avail:
172 out.append('\nTools and libraries NOT available at test time:\n')
173 out.append('\nTools and libraries NOT available at test time:\n')
173 not_avail.sort()
174 not_avail.sort()
174 out.append(' ' + ' '.join(not_avail)+'\n')
175 out.append(' ' + ' '.join(not_avail)+'\n')
175
176
176 return ''.join(out)
177 return ''.join(out)
177
178
178
179
179 def make_exclude():
180 def make_exclude():
180 """Make patterns of modules and packages to exclude from testing.
181 """Make patterns of modules and packages to exclude from testing.
181
182
182 For the IPythonDoctest plugin, we need to exclude certain patterns that
183 For the IPythonDoctest plugin, we need to exclude certain patterns that
183 cause testing problems. We should strive to minimize the number of
184 cause testing problems. We should strive to minimize the number of
184 skipped modules, since this means untested code.
185 skipped modules, since this means untested code.
185
186
186 These modules and packages will NOT get scanned by nose at all for tests.
187 These modules and packages will NOT get scanned by nose at all for tests.
187 """
188 """
188 # Simple utility to make IPython paths more readably, we need a lot of
189 # Simple utility to make IPython paths more readably, we need a lot of
189 # these below
190 # these below
190 ipjoin = lambda *paths: pjoin('IPython', *paths)
191 ipjoin = lambda *paths: pjoin('IPython', *paths)
191
192
192 exclusions = [ipjoin('external'),
193 exclusions = [ipjoin('external'),
193 pjoin('IPython_doctest_plugin'),
194 pjoin('IPython_doctest_plugin'),
194 ipjoin('quarantine'),
195 ipjoin('quarantine'),
195 ipjoin('deathrow'),
196 ipjoin('deathrow'),
196 ipjoin('testing', 'attic'),
197 ipjoin('testing', 'attic'),
197 # This guy is probably attic material
198 # This guy is probably attic material
198 ipjoin('testing', 'mkdoctests'),
199 ipjoin('testing', 'mkdoctests'),
199 # Testing inputhook will need a lot of thought, to figure out
200 # Testing inputhook will need a lot of thought, to figure out
200 # how to have tests that don't lock up with the gui event
201 # how to have tests that don't lock up with the gui event
201 # loops in the picture
202 # loops in the picture
202 ipjoin('lib', 'inputhook'),
203 ipjoin('lib', 'inputhook'),
203 # Config files aren't really importable stand-alone
204 # Config files aren't really importable stand-alone
204 ipjoin('config', 'default'),
205 ipjoin('config', 'default'),
205 ipjoin('config', 'profile'),
206 ipjoin('config', 'profile'),
206 ]
207 ]
207
208 if not have['sqlite3']:
209 exclusions.append(ipjoin('core', 'tests', 'test_history'))
210 exclusions.append(ipjoin('core', 'history'))
208 if not have['wx']:
211 if not have['wx']:
209 exclusions.append(ipjoin('lib', 'inputhookwx'))
212 exclusions.append(ipjoin('lib', 'inputhookwx'))
210
213
211 # We do this unconditionally, so that the test suite doesn't import
214 # We do this unconditionally, so that the test suite doesn't import
212 # gtk, changing the default encoding and masking some unicode bugs.
215 # gtk, changing the default encoding and masking some unicode bugs.
213 exclusions.append(ipjoin('lib', 'inputhookgtk'))
216 exclusions.append(ipjoin('lib', 'inputhookgtk'))
214
217
215 # These have to be skipped on win32 because the use echo, rm, cd, etc.
218 # These have to be skipped on win32 because the use echo, rm, cd, etc.
216 # See ticket https://github.com/ipython/ipython/issues/87
219 # See ticket https://github.com/ipython/ipython/issues/87
217 if sys.platform == 'win32':
220 if sys.platform == 'win32':
218 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
221 exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip'))
219 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
222 exclusions.append(ipjoin('testing', 'plugin', 'dtexample'))
220
223
221 if not have['pexpect']:
224 if not have['pexpect']:
222 exclusions.extend([ipjoin('scripts', 'irunner'),
225 exclusions.extend([ipjoin('scripts', 'irunner'),
223 ipjoin('lib', 'irunner'),
226 ipjoin('lib', 'irunner'),
224 ipjoin('lib', 'tests', 'test_irunner')])
227 ipjoin('lib', 'tests', 'test_irunner')])
225
228
226 if not have['zmq']:
229 if not have['zmq']:
227 exclusions.append(ipjoin('zmq'))
230 exclusions.append(ipjoin('zmq'))
228 exclusions.append(ipjoin('frontend', 'qt'))
231 exclusions.append(ipjoin('frontend', 'qt'))
229 exclusions.append(ipjoin('parallel'))
232 exclusions.append(ipjoin('parallel'))
230 elif not have['qt']:
233 elif not have['qt']:
231 exclusions.append(ipjoin('frontend', 'qt'))
234 exclusions.append(ipjoin('frontend', 'qt'))
232
235
233 if not have['pymongo']:
236 if not have['pymongo']:
234 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
237 exclusions.append(ipjoin('parallel', 'controller', 'mongodb'))
235 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
238 exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb'))
236
239
237 if not have['matplotlib']:
240 if not have['matplotlib']:
238 exclusions.extend([ipjoin('lib', 'pylabtools'),
241 exclusions.extend([ipjoin('lib', 'pylabtools'),
239 ipjoin('lib', 'tests', 'test_pylabtools')])
242 ipjoin('lib', 'tests', 'test_pylabtools')])
240
243
241 if not have['tornado']:
244 if not have['tornado']:
242 exclusions.append(ipjoin('frontend', 'html'))
245 exclusions.append(ipjoin('frontend', 'html'))
243
246
244 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
247 # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
245 if sys.platform == 'win32':
248 if sys.platform == 'win32':
246 exclusions = [s.replace('\\','\\\\') for s in exclusions]
249 exclusions = [s.replace('\\','\\\\') for s in exclusions]
247
250
248 return exclusions
251 return exclusions
249
252
250
253
251 class IPTester(object):
254 class IPTester(object):
252 """Call that calls iptest or trial in a subprocess.
255 """Call that calls iptest or trial in a subprocess.
253 """
256 """
254 #: string, name of test runner that will be called
257 #: string, name of test runner that will be called
255 runner = None
258 runner = None
256 #: list, parameters for test runner
259 #: list, parameters for test runner
257 params = None
260 params = None
258 #: list, arguments of system call to be made to call test runner
261 #: list, arguments of system call to be made to call test runner
259 call_args = None
262 call_args = None
260 #: list, process ids of subprocesses we start (for cleanup)
263 #: list, process ids of subprocesses we start (for cleanup)
261 pids = None
264 pids = None
262
265
263 def __init__(self, runner='iptest', params=None):
266 def __init__(self, runner='iptest', params=None):
264 """Create new test runner."""
267 """Create new test runner."""
265 p = os.path
268 p = os.path
266 if runner == 'iptest':
269 if runner == 'iptest':
267 iptest_app = get_ipython_module_path('IPython.testing.iptest')
270 iptest_app = get_ipython_module_path('IPython.testing.iptest')
268 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
271 self.runner = pycmd2argv(iptest_app) + sys.argv[1:]
269 else:
272 else:
270 raise Exception('Not a valid test runner: %s' % repr(runner))
273 raise Exception('Not a valid test runner: %s' % repr(runner))
271 if params is None:
274 if params is None:
272 params = []
275 params = []
273 if isinstance(params, str):
276 if isinstance(params, str):
274 params = [params]
277 params = [params]
275 self.params = params
278 self.params = params
276
279
277 # Assemble call
280 # Assemble call
278 self.call_args = self.runner+self.params
281 self.call_args = self.runner+self.params
279
282
280 # Store pids of anything we start to clean up on deletion, if possible
283 # Store pids of anything we start to clean up on deletion, if possible
281 # (on posix only, since win32 has no os.kill)
284 # (on posix only, since win32 has no os.kill)
282 self.pids = []
285 self.pids = []
283
286
284 if sys.platform == 'win32':
287 if sys.platform == 'win32':
285 def _run_cmd(self):
288 def _run_cmd(self):
286 # On Windows, use os.system instead of subprocess.call, because I
289 # On Windows, use os.system instead of subprocess.call, because I
287 # was having problems with subprocess and I just don't know enough
290 # was having problems with subprocess and I just don't know enough
288 # about win32 to debug this reliably. Os.system may be the 'old
291 # about win32 to debug this reliably. Os.system may be the 'old
289 # fashioned' way to do it, but it works just fine. If someone
292 # fashioned' way to do it, but it works just fine. If someone
290 # later can clean this up that's fine, as long as the tests run
293 # later can clean this up that's fine, as long as the tests run
291 # reliably in win32.
294 # reliably in win32.
292 # What types of problems are you having. They may be related to
295 # What types of problems are you having. They may be related to
293 # running Python in unboffered mode. BG.
296 # running Python in unboffered mode. BG.
294 return os.system(' '.join(self.call_args))
297 return os.system(' '.join(self.call_args))
295 else:
298 else:
296 def _run_cmd(self):
299 def _run_cmd(self):
297 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
300 # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
298 subp = subprocess.Popen(self.call_args)
301 subp = subprocess.Popen(self.call_args)
299 self.pids.append(subp.pid)
302 self.pids.append(subp.pid)
300 # If this fails, the pid will be left in self.pids and cleaned up
303 # If this fails, the pid will be left in self.pids and cleaned up
301 # later, but if the wait call succeeds, then we can clear the
304 # later, but if the wait call succeeds, then we can clear the
302 # stored pid.
305 # stored pid.
303 retcode = subp.wait()
306 retcode = subp.wait()
304 self.pids.pop()
307 self.pids.pop()
305 return retcode
308 return retcode
306
309
307 def run(self):
310 def run(self):
308 """Run the stored commands"""
311 """Run the stored commands"""
309 try:
312 try:
310 return self._run_cmd()
313 return self._run_cmd()
311 except:
314 except:
312 import traceback
315 import traceback
313 traceback.print_exc()
316 traceback.print_exc()
314 return 1 # signal failure
317 return 1 # signal failure
315
318
316 def __del__(self):
319 def __del__(self):
317 """Cleanup on exit by killing any leftover processes."""
320 """Cleanup on exit by killing any leftover processes."""
318
321
319 if not hasattr(os, 'kill'):
322 if not hasattr(os, 'kill'):
320 return
323 return
321
324
322 for pid in self.pids:
325 for pid in self.pids:
323 try:
326 try:
324 print 'Cleaning stale PID:', pid
327 print 'Cleaning stale PID:', pid
325 os.kill(pid, signal.SIGKILL)
328 os.kill(pid, signal.SIGKILL)
326 except OSError:
329 except OSError:
327 # This is just a best effort, if we fail or the process was
330 # This is just a best effort, if we fail or the process was
328 # really gone, ignore it.
331 # really gone, ignore it.
329 pass
332 pass
330
333
331
334
332 def make_runners():
335 def make_runners():
333 """Define the top-level packages that need to be tested.
336 """Define the top-level packages that need to be tested.
334 """
337 """
335
338
336 # Packages to be tested via nose, that only depend on the stdlib
339 # Packages to be tested via nose, that only depend on the stdlib
337 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
340 nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib',
338 'scripts', 'testing', 'utils', 'nbformat' ]
341 'scripts', 'testing', 'utils', 'nbformat' ]
339
342
340 if have['zmq']:
343 if have['zmq']:
341 nose_pkg_names.append('parallel')
344 nose_pkg_names.append('parallel')
342
345
343 # For debugging this code, only load quick stuff
346 # For debugging this code, only load quick stuff
344 #nose_pkg_names = ['core', 'extensions'] # dbg
347 #nose_pkg_names = ['core', 'extensions'] # dbg
345
348
346 # Make fully qualified package names prepending 'IPython.' to our name lists
349 # Make fully qualified package names prepending 'IPython.' to our name lists
347 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
350 nose_packages = ['IPython.%s' % m for m in nose_pkg_names ]
348
351
349 # Make runners
352 # Make runners
350 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
353 runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ]
351
354
352 return runners
355 return runners
353
356
354
357
355 def run_iptest():
358 def run_iptest():
356 """Run the IPython test suite using nose.
359 """Run the IPython test suite using nose.
357
360
358 This function is called when this script is **not** called with the form
361 This function is called when this script is **not** called with the form
359 `iptest all`. It simply calls nose with appropriate command line flags
362 `iptest all`. It simply calls nose with appropriate command line flags
360 and accepts all of the standard nose arguments.
363 and accepts all of the standard nose arguments.
361 """
364 """
362
365
363 warnings.filterwarnings('ignore',
366 warnings.filterwarnings('ignore',
364 'This will be removed soon. Use IPython.testing.util instead')
367 'This will be removed soon. Use IPython.testing.util instead')
365
368
366 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
369 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
367
370
368 # Loading ipdoctest causes problems with Twisted, but
371 # Loading ipdoctest causes problems with Twisted, but
369 # our test suite runner now separates things and runs
372 # our test suite runner now separates things and runs
370 # all Twisted tests with trial.
373 # all Twisted tests with trial.
371 '--with-ipdoctest',
374 '--with-ipdoctest',
372 '--ipdoctest-tests','--ipdoctest-extension=txt',
375 '--ipdoctest-tests','--ipdoctest-extension=txt',
373
376
374 # We add --exe because of setuptools' imbecility (it
377 # We add --exe because of setuptools' imbecility (it
375 # blindly does chmod +x on ALL files). Nose does the
378 # blindly does chmod +x on ALL files). Nose does the
376 # right thing and it tries to avoid executables,
379 # right thing and it tries to avoid executables,
377 # setuptools unfortunately forces our hand here. This
380 # setuptools unfortunately forces our hand here. This
378 # has been discussed on the distutils list and the
381 # has been discussed on the distutils list and the
379 # setuptools devs refuse to fix this problem!
382 # setuptools devs refuse to fix this problem!
380 '--exe',
383 '--exe',
381 ]
384 ]
382
385
383 if nose.__version__ >= '0.11':
386 if nose.__version__ >= '0.11':
384 # I don't fully understand why we need this one, but depending on what
387 # I don't fully understand why we need this one, but depending on what
385 # directory the test suite is run from, if we don't give it, 0 tests
388 # directory the test suite is run from, if we don't give it, 0 tests
386 # get run. Specifically, if the test suite is run from the source dir
389 # get run. Specifically, if the test suite is run from the source dir
387 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
390 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
388 # even if the same call done in this directory works fine). It appears
391 # even if the same call done in this directory works fine). It appears
389 # that if the requested package is in the current dir, nose bails early
392 # that if the requested package is in the current dir, nose bails early
390 # by default. Since it's otherwise harmless, leave it in by default
393 # by default. Since it's otherwise harmless, leave it in by default
391 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
394 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
392 argv.append('--traverse-namespace')
395 argv.append('--traverse-namespace')
393
396
394 # use our plugin for doctesting. It will remove the standard doctest plugin
397 # use our plugin for doctesting. It will remove the standard doctest plugin
395 # if it finds it enabled
398 # if it finds it enabled
396 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
399 plugins = [IPythonDoctest(make_exclude()), KnownFailure()]
397 # We need a global ipython running in this process
400 # We need a global ipython running in this process
398 globalipapp.start_ipython()
401 globalipapp.start_ipython()
399 # Now nose can run
402 # Now nose can run
400 TestProgram(argv=argv, addplugins=plugins)
403 TestProgram(argv=argv, addplugins=plugins)
401
404
402
405
403 def run_iptestall():
406 def run_iptestall():
404 """Run the entire IPython test suite by calling nose and trial.
407 """Run the entire IPython test suite by calling nose and trial.
405
408
406 This function constructs :class:`IPTester` instances for all IPython
409 This function constructs :class:`IPTester` instances for all IPython
407 modules and package and then runs each of them. This causes the modules
410 modules and package and then runs each of them. This causes the modules
408 and packages of IPython to be tested each in their own subprocess using
411 and packages of IPython to be tested each in their own subprocess using
409 nose or twisted.trial appropriately.
412 nose or twisted.trial appropriately.
410 """
413 """
411
414
412 runners = make_runners()
415 runners = make_runners()
413
416
414 # Run the test runners in a temporary dir so we can nuke it when finished
417 # Run the test runners in a temporary dir so we can nuke it when finished
415 # to clean up any junk files left over by accident. This also makes it
418 # to clean up any junk files left over by accident. This also makes it
416 # robust against being run in non-writeable directories by mistake, as the
419 # robust against being run in non-writeable directories by mistake, as the
417 # temp dir will always be user-writeable.
420 # temp dir will always be user-writeable.
418 curdir = os.getcwdu()
421 curdir = os.getcwdu()
419 testdir = tempfile.gettempdir()
422 testdir = tempfile.gettempdir()
420 os.chdir(testdir)
423 os.chdir(testdir)
421
424
422 # Run all test runners, tracking execution time
425 # Run all test runners, tracking execution time
423 failed = []
426 failed = []
424 t_start = time.time()
427 t_start = time.time()
425 try:
428 try:
426 for (name, runner) in runners:
429 for (name, runner) in runners:
427 print '*'*70
430 print '*'*70
428 print 'IPython test group:',name
431 print 'IPython test group:',name
429 res = runner.run()
432 res = runner.run()
430 if res:
433 if res:
431 failed.append( (name, runner) )
434 failed.append( (name, runner) )
432 finally:
435 finally:
433 os.chdir(curdir)
436 os.chdir(curdir)
434 t_end = time.time()
437 t_end = time.time()
435 t_tests = t_end - t_start
438 t_tests = t_end - t_start
436 nrunners = len(runners)
439 nrunners = len(runners)
437 nfail = len(failed)
440 nfail = len(failed)
438 # summarize results
441 # summarize results
439 print
442 print
440 print '*'*70
443 print '*'*70
441 print 'Test suite completed for system with the following information:'
444 print 'Test suite completed for system with the following information:'
442 print report()
445 print report()
443 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
446 print 'Ran %s test groups in %.3fs' % (nrunners, t_tests)
444 print
447 print
445 print 'Status:'
448 print 'Status:'
446 if not failed:
449 if not failed:
447 print 'OK'
450 print 'OK'
448 else:
451 else:
449 # If anything went wrong, point out what command to rerun manually to
452 # If anything went wrong, point out what command to rerun manually to
450 # see the actual errors and individual summary
453 # see the actual errors and individual summary
451 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
454 print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners)
452 for name, failed_runner in failed:
455 for name, failed_runner in failed:
453 print '-'*40
456 print '-'*40
454 print 'Runner failed:',name
457 print 'Runner failed:',name
455 print 'You may wish to rerun this one individually, with:'
458 print 'You may wish to rerun this one individually, with:'
456 print ' '.join(failed_runner.call_args)
459 print ' '.join(failed_runner.call_args)
457 print
460 print
458 # Ensure that our exit code indicates failure
461 # Ensure that our exit code indicates failure
459 sys.exit(1)
462 sys.exit(1)
460
463
461
464
462 def main():
465 def main():
463 for arg in sys.argv[1:]:
466 for arg in sys.argv[1:]:
464 if arg.startswith('IPython'):
467 if arg.startswith('IPython'):
465 # This is in-process
468 # This is in-process
466 run_iptest()
469 run_iptest()
467 else:
470 else:
468 # This starts subprocesses
471 # This starts subprocesses
469 run_iptestall()
472 run_iptestall()
470
473
471
474
472 if __name__ == '__main__':
475 if __name__ == '__main__':
473 main()
476 main()
General Comments 0
You need to be logged in to leave comments. Login now