##// END OF EJS Templates
Merging Fernando's fixes from his trunk-dev and fixing testing things....
Brian Granger -
r1973:5d4c3105 merge
parent child Browse files
Show More
@@ -0,0 +1,94 b''
1 """Tests for the ipdoctest machinery itself.
2
3 Note: in a file named test_X, functions whose only test is their docstring (as
4 a doctest) and which have no test functionality of their own, should be called
5 'doctest_foo' instead of 'test_foo', otherwise they get double-counted (the
6 empty function call is counted as a test, which just inflates tests numbers
7 artificially).
8 """
9
10 def doctest_simple():
11 """ipdoctest must handle simple inputs
12
13 In [1]: 1
14 Out[1]: 1
15
16 In [2]: print 1
17 1
18 """
19
20
21 def doctest_run_builtins():
22 """Check that %run doesn't damage __builtins__ via a doctest.
23
24 This is similar to the test_run_builtins, but I want *both* forms of the
25 test to catch any possible glitches in our testing machinery, since that
26 modifies %run somewhat. So for this, we have both a normal test (below)
27 and a doctest (this one).
28
29 In [1]: import tempfile
30
31 In [3]: f = tempfile.NamedTemporaryFile()
32
33 In [4]: f.write('pass\\n')
34
35 In [5]: f.flush()
36
37 In [7]: %run $f.name
38 """
39
40 def doctest_multiline1():
41 """The ipdoctest machinery must handle multiline examples gracefully.
42
43 In [2]: for i in range(10):
44 ...: print i,
45 ...:
46 0 1 2 3 4 5 6 7 8 9
47 """
48
49
50 def doctest_multiline2():
51 """Multiline examples that define functions and print output.
52
53 In [7]: def f(x):
54 ...: return x+1
55 ...:
56
57 In [8]: f(1)
58 Out[8]: 2
59
60 In [9]: def g(x):
61 ...: print 'x is:',x
62 ...:
63
64 In [10]: g(1)
65 x is: 1
66
67 In [11]: g('hello')
68 x is: hello
69 """
70
71
72 def doctest_multiline3():
73 """Multiline examples with blank lines.
74
75 In [12]: def h(x):
76 ....: if x>1:
77 ....: return x**2
78 ....: # To leave a blank line in the input, you must mark it
79 ....: # with a comment character:
80 ....: #
81 ....: # otherwise the doctest parser gets confused.
82 ....: else:
83 ....: return -1
84 ....:
85
86 In [13]: h(5)
87 Out[13]: 25
88
89 In [14]: h(1)
90 Out[14]: -1
91
92 In [15]: h(0)
93 Out[15]: -1
94 """
@@ -0,0 +1,90 b''
1 """Generic testing tools that do NOT depend on Twisted.
2
3 In particular, this module exposes a set of top-level assert* functions that
4 can be used in place of nose.tools.assert* in method generators (the ones in
5 nose can not, at least as of nose 0.10.4).
6
7 Note: our testing package contains testing.util, which does depend on Twisted
8 and provides utilities for tests that manage Deferreds. All testing support
9 tools that only depend on nose, IPython or the standard library should go here
10 instead.
11
12
13 Authors
14 -------
15 - Fernando Perez <Fernando.Perez@berkeley.edu>
16 """
17
18 #*****************************************************************************
19 # Copyright (C) 2009 The IPython Development Team
20 #
21 # Distributed under the terms of the BSD License. The full license is in
22 # the file COPYING, distributed as part of this software.
23 #*****************************************************************************
24
25 #-----------------------------------------------------------------------------
26 # Required modules and packages
27 #-----------------------------------------------------------------------------
28
29 # Standard Python lib
30 import os
31 import sys
32
33 # Third-party
34 import nose.tools as nt
35
36 # From this project
37 from IPython.tools import utils
38
39 #-----------------------------------------------------------------------------
40 # Globals
41 #-----------------------------------------------------------------------------
42
43 # Make a bunch of nose.tools assert wrappers that can be used in test
44 # generators. This will expose an assert* function for each one in nose.tools.
45
46 _tpl = """
47 def %(name)s(*a,**kw):
48 return nt.%(name)s(*a,**kw)
49 """
50
51 for _x in [a for a in dir(nt) if a.startswith('assert')]:
52 exec _tpl % dict(name=_x)
53
54 #-----------------------------------------------------------------------------
55 # Functions and classes
56 #-----------------------------------------------------------------------------
57
58 def full_path(startPath,files):
59 """Make full paths for all the listed files, based on startPath.
60
61 Only the base part of startPath is kept, since this routine is typically
62 used with a script's __file__ variable as startPath. The base of startPath
63 is then prepended to all the listed files, forming the output list.
64
65 Parameters
66 ----------
67 startPath : string
68 Initial path to use as the base for the results. This path is split
69 using os.path.split() and only its first component is kept.
70
71 files : string or list
72 One or more files.
73
74 Examples
75 --------
76
77 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
78 ['/foo/a.txt', '/foo/b.txt']
79
80 >>> full_path('/foo',['a.txt','b.txt'])
81 ['/a.txt', '/b.txt']
82
83 If a single file is given, the output is still a list:
84 >>> full_path('/foo','a.txt')
85 ['/a.txt']
86 """
87
88 files = utils.list_strings(files)
89 base = os.path.split(startPath)[0]
90 return [ os.path.join(base,f) for f in files ]
@@ -1,3446 +1,3456 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Magic functions for InteractiveShell.
3 3 """
4 4
5 5 #*****************************************************************************
6 6 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
7 7 # Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #*****************************************************************************
12 12
13 13 #****************************************************************************
14 14 # Modules and globals
15 15
16 16 # Python standard modules
17 17 import __builtin__
18 18 import bdb
19 19 import inspect
20 20 import os
21 21 import pdb
22 22 import pydoc
23 23 import sys
24 24 import re
25 25 import tempfile
26 26 import time
27 27 import cPickle as pickle
28 28 import textwrap
29 29 from cStringIO import StringIO
30 30 from getopt import getopt,GetoptError
31 31 from pprint import pprint, pformat
32 32
33 33 # cProfile was added in Python2.5
34 34 try:
35 35 import cProfile as profile
36 36 import pstats
37 37 except ImportError:
38 38 # profile isn't bundled by default in Debian for license reasons
39 39 try:
40 40 import profile,pstats
41 41 except ImportError:
42 42 profile = pstats = None
43 43
44 44 # Homebrewed
45 45 import IPython
46 46 from IPython import Debugger, OInspect, wildcard
47 47 from IPython.FakeModule import FakeModule
48 48 from IPython.Itpl import Itpl, itpl, printpl,itplns
49 49 from IPython.PyColorize import Parser
50 50 from IPython.ipstruct import Struct
51 51 from IPython.macro import Macro
52 52 from IPython.genutils import *
53 53 from IPython import platutils
54 54 import IPython.generics
55 55 import IPython.ipapi
56 56 from IPython.ipapi import UsageError
57 57 from IPython.testing import decorators as testdec
58 58
59 59 #***************************************************************************
60 60 # Utility functions
61 61 def on_off(tag):
62 62 """Return an ON/OFF string for a 1/0 input. Simple utility function."""
63 63 return ['OFF','ON'][tag]
64 64
65 65 class Bunch: pass
66 66
67 67 def compress_dhist(dh):
68 68 head, tail = dh[:-10], dh[-10:]
69 69
70 70 newhead = []
71 71 done = set()
72 72 for h in head:
73 73 if h in done:
74 74 continue
75 75 newhead.append(h)
76 76 done.add(h)
77 77
78 78 return newhead + tail
79 79
80 80
81 81 #***************************************************************************
82 82 # Main class implementing Magic functionality
83 83 class Magic:
84 84 """Magic functions for InteractiveShell.
85 85
86 86 Shell functions which can be reached as %function_name. All magic
87 87 functions should accept a string, which they can parse for their own
88 88 needs. This can make some functions easier to type, eg `%cd ../`
89 89 vs. `%cd("../")`
90 90
91 91 ALL definitions MUST begin with the prefix magic_. The user won't need it
92 92 at the command line, but it is is needed in the definition. """
93 93
94 94 # class globals
95 95 auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
96 96 'Automagic is ON, % prefix NOT needed for magic functions.']
97 97
98 98 #......................................................................
99 99 # some utility functions
100 100
101 101 def __init__(self,shell):
102 102
103 103 self.options_table = {}
104 104 if profile is None:
105 105 self.magic_prun = self.profile_missing_notice
106 106 self.shell = shell
107 107
108 108 # namespace for holding state we may need
109 109 self._magic_state = Bunch()
110 110
111 111 def profile_missing_notice(self, *args, **kwargs):
112 112 error("""\
113 113 The profile module could not be found. It has been removed from the standard
114 114 python packages because of its non-free license. To use profiling, install the
115 115 python-profiler package from non-free.""")
116 116
117 117 def default_option(self,fn,optstr):
118 118 """Make an entry in the options_table for fn, with value optstr"""
119 119
120 120 if fn not in self.lsmagic():
121 121 error("%s is not a magic function" % fn)
122 122 self.options_table[fn] = optstr
123 123
124 124 def lsmagic(self):
125 125 """Return a list of currently available magic functions.
126 126
127 127 Gives a list of the bare names after mangling (['ls','cd', ...], not
128 128 ['magic_ls','magic_cd',...]"""
129 129
130 130 # FIXME. This needs a cleanup, in the way the magics list is built.
131 131
132 132 # magics in class definition
133 133 class_magic = lambda fn: fn.startswith('magic_') and \
134 134 callable(Magic.__dict__[fn])
135 135 # in instance namespace (run-time user additions)
136 136 inst_magic = lambda fn: fn.startswith('magic_') and \
137 137 callable(self.__dict__[fn])
138 138 # and bound magics by user (so they can access self):
139 139 inst_bound_magic = lambda fn: fn.startswith('magic_') and \
140 140 callable(self.__class__.__dict__[fn])
141 141 magics = filter(class_magic,Magic.__dict__.keys()) + \
142 142 filter(inst_magic,self.__dict__.keys()) + \
143 143 filter(inst_bound_magic,self.__class__.__dict__.keys())
144 144 out = []
145 145 for fn in set(magics):
146 146 out.append(fn.replace('magic_','',1))
147 147 out.sort()
148 148 return out
149 149
150 150 def extract_input_slices(self,slices,raw=False):
151 151 """Return as a string a set of input history slices.
152 152
153 153 Inputs:
154 154
155 155 - slices: the set of slices is given as a list of strings (like
156 156 ['1','4:8','9'], since this function is for use by magic functions
157 157 which get their arguments as strings.
158 158
159 159 Optional inputs:
160 160
161 161 - raw(False): by default, the processed input is used. If this is
162 162 true, the raw input history is used instead.
163 163
164 164 Note that slices can be called with two notations:
165 165
166 166 N:M -> standard python form, means including items N...(M-1).
167 167
168 168 N-M -> include items N..M (closed endpoint)."""
169 169
170 170 if raw:
171 171 hist = self.shell.input_hist_raw
172 172 else:
173 173 hist = self.shell.input_hist
174 174
175 175 cmds = []
176 176 for chunk in slices:
177 177 if ':' in chunk:
178 178 ini,fin = map(int,chunk.split(':'))
179 179 elif '-' in chunk:
180 180 ini,fin = map(int,chunk.split('-'))
181 181 fin += 1
182 182 else:
183 183 ini = int(chunk)
184 184 fin = ini+1
185 185 cmds.append(hist[ini:fin])
186 186 return cmds
187 187
188 188 def _ofind(self, oname, namespaces=None):
189 189 """Find an object in the available namespaces.
190 190
191 191 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
192 192
193 193 Has special code to detect magic functions.
194 194 """
195 195
196 196 oname = oname.strip()
197 197
198 198 alias_ns = None
199 199 if namespaces is None:
200 200 # Namespaces to search in:
201 201 # Put them in a list. The order is important so that we
202 202 # find things in the same order that Python finds them.
203 203 namespaces = [ ('Interactive', self.shell.user_ns),
204 204 ('IPython internal', self.shell.internal_ns),
205 205 ('Python builtin', __builtin__.__dict__),
206 206 ('Alias', self.shell.alias_table),
207 207 ]
208 208 alias_ns = self.shell.alias_table
209 209
210 210 # initialize results to 'null'
211 211 found = 0; obj = None; ospace = None; ds = None;
212 212 ismagic = 0; isalias = 0; parent = None
213 213
214 214 # Look for the given name by splitting it in parts. If the head is
215 215 # found, then we look for all the remaining parts as members, and only
216 216 # declare success if we can find them all.
217 217 oname_parts = oname.split('.')
218 218 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
219 219 for nsname,ns in namespaces:
220 220 try:
221 221 obj = ns[oname_head]
222 222 except KeyError:
223 223 continue
224 224 else:
225 225 #print 'oname_rest:', oname_rest # dbg
226 226 for part in oname_rest:
227 227 try:
228 228 parent = obj
229 229 obj = getattr(obj,part)
230 230 except:
231 231 # Blanket except b/c some badly implemented objects
232 232 # allow __getattr__ to raise exceptions other than
233 233 # AttributeError, which then crashes IPython.
234 234 break
235 235 else:
236 236 # If we finish the for loop (no break), we got all members
237 237 found = 1
238 238 ospace = nsname
239 239 if ns == alias_ns:
240 240 isalias = 1
241 241 break # namespace loop
242 242
243 243 # Try to see if it's magic
244 244 if not found:
245 245 if oname.startswith(self.shell.ESC_MAGIC):
246 246 oname = oname[1:]
247 247 obj = getattr(self,'magic_'+oname,None)
248 248 if obj is not None:
249 249 found = 1
250 250 ospace = 'IPython internal'
251 251 ismagic = 1
252 252
253 253 # Last try: special-case some literals like '', [], {}, etc:
254 254 if not found and oname_head in ["''",'""','[]','{}','()']:
255 255 obj = eval(oname_head)
256 256 found = 1
257 257 ospace = 'Interactive'
258 258
259 259 return {'found':found, 'obj':obj, 'namespace':ospace,
260 260 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
261 261
262 262 def arg_err(self,func):
263 263 """Print docstring if incorrect arguments were passed"""
264 264 print 'Error in arguments:'
265 265 print OInspect.getdoc(func)
266 266
267 267 def format_latex(self,strng):
268 268 """Format a string for latex inclusion."""
269 269
270 270 # Characters that need to be escaped for latex:
271 271 escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
272 272 # Magic command names as headers:
273 273 cmd_name_re = re.compile(r'^(%s.*?):' % self.shell.ESC_MAGIC,
274 274 re.MULTILINE)
275 275 # Magic commands
276 276 cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % self.shell.ESC_MAGIC,
277 277 re.MULTILINE)
278 278 # Paragraph continue
279 279 par_re = re.compile(r'\\$',re.MULTILINE)
280 280
281 281 # The "\n" symbol
282 282 newline_re = re.compile(r'\\n')
283 283
284 284 # Now build the string for output:
285 285 #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
286 286 strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
287 287 strng)
288 288 strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
289 289 strng = par_re.sub(r'\\\\',strng)
290 290 strng = escape_re.sub(r'\\\1',strng)
291 291 strng = newline_re.sub(r'\\textbackslash{}n',strng)
292 292 return strng
293 293
294 294 def format_screen(self,strng):
295 295 """Format a string for screen printing.
296 296
297 297 This removes some latex-type format codes."""
298 298 # Paragraph continue
299 299 par_re = re.compile(r'\\$',re.MULTILINE)
300 300 strng = par_re.sub('',strng)
301 301 return strng
302 302
303 303 def parse_options(self,arg_str,opt_str,*long_opts,**kw):
304 304 """Parse options passed to an argument string.
305 305
306 306 The interface is similar to that of getopt(), but it returns back a
307 307 Struct with the options as keys and the stripped argument string still
308 308 as a string.
309 309
310 310 arg_str is quoted as a true sys.argv vector by using shlex.split.
311 311 This allows us to easily expand variables, glob files, quote
312 312 arguments, etc.
313 313
314 314 Options:
315 315 -mode: default 'string'. If given as 'list', the argument string is
316 316 returned as a list (split on whitespace) instead of a string.
317 317
318 318 -list_all: put all option values in lists. Normally only options
319 319 appearing more than once are put in a list.
320 320
321 321 -posix (True): whether to split the input line in POSIX mode or not,
322 322 as per the conventions outlined in the shlex module from the
323 323 standard library."""
324 324
325 325 # inject default options at the beginning of the input line
326 326 caller = sys._getframe(1).f_code.co_name.replace('magic_','')
327 327 arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
328 328
329 329 mode = kw.get('mode','string')
330 330 if mode not in ['string','list']:
331 331 raise ValueError,'incorrect mode given: %s' % mode
332 332 # Get options
333 333 list_all = kw.get('list_all',0)
334 334 posix = kw.get('posix',True)
335 335
336 336 # Check if we have more than one argument to warrant extra processing:
337 337 odict = {} # Dictionary with options
338 338 args = arg_str.split()
339 339 if len(args) >= 1:
340 340 # If the list of inputs only has 0 or 1 thing in it, there's no
341 341 # need to look for options
342 342 argv = arg_split(arg_str,posix)
343 343 # Do regular option processing
344 344 try:
345 345 opts,args = getopt(argv,opt_str,*long_opts)
346 346 except GetoptError,e:
347 347 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
348 348 " ".join(long_opts)))
349 349 for o,a in opts:
350 350 if o.startswith('--'):
351 351 o = o[2:]
352 352 else:
353 353 o = o[1:]
354 354 try:
355 355 odict[o].append(a)
356 356 except AttributeError:
357 357 odict[o] = [odict[o],a]
358 358 except KeyError:
359 359 if list_all:
360 360 odict[o] = [a]
361 361 else:
362 362 odict[o] = a
363 363
364 364 # Prepare opts,args for return
365 365 opts = Struct(odict)
366 366 if mode == 'string':
367 367 args = ' '.join(args)
368 368
369 369 return opts,args
370 370
371 371 #......................................................................
372 372 # And now the actual magic functions
373 373
374 374 # Functions for IPython shell work (vars,funcs, config, etc)
375 375 def magic_lsmagic(self, parameter_s = ''):
376 376 """List currently available magic functions."""
377 377 mesc = self.shell.ESC_MAGIC
378 378 print 'Available magic functions:\n'+mesc+\
379 379 (' '+mesc).join(self.lsmagic())
380 380 print '\n' + Magic.auto_status[self.shell.rc.automagic]
381 381 return None
382 382
383 383 def magic_magic(self, parameter_s = ''):
384 384 """Print information about the magic function system.
385 385
386 386 Supported formats: -latex, -brief, -rest
387 387 """
388 388
389 389 mode = ''
390 390 try:
391 391 if parameter_s.split()[0] == '-latex':
392 392 mode = 'latex'
393 393 if parameter_s.split()[0] == '-brief':
394 394 mode = 'brief'
395 395 if parameter_s.split()[0] == '-rest':
396 396 mode = 'rest'
397 397 rest_docs = []
398 398 except:
399 399 pass
400 400
401 401 magic_docs = []
402 402 for fname in self.lsmagic():
403 403 mname = 'magic_' + fname
404 404 for space in (Magic,self,self.__class__):
405 405 try:
406 406 fn = space.__dict__[mname]
407 407 except KeyError:
408 408 pass
409 409 else:
410 410 break
411 411 if mode == 'brief':
412 412 # only first line
413 413 if fn.__doc__:
414 414 fndoc = fn.__doc__.split('\n',1)[0]
415 415 else:
416 416 fndoc = 'No documentation'
417 417 else:
418 418 if fn.__doc__:
419 419 fndoc = fn.__doc__.rstrip()
420 420 else:
421 421 fndoc = 'No documentation'
422 422
423 423
424 424 if mode == 'rest':
425 425 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(self.shell.ESC_MAGIC,
426 426 fname,fndoc))
427 427
428 428 else:
429 429 magic_docs.append('%s%s:\n\t%s\n' %(self.shell.ESC_MAGIC,
430 430 fname,fndoc))
431 431
432 432 magic_docs = ''.join(magic_docs)
433 433
434 434 if mode == 'rest':
435 435 return "".join(rest_docs)
436 436
437 437 if mode == 'latex':
438 438 print self.format_latex(magic_docs)
439 439 return
440 440 else:
441 441 magic_docs = self.format_screen(magic_docs)
442 442 if mode == 'brief':
443 443 return magic_docs
444 444
445 445 outmsg = """
446 446 IPython's 'magic' functions
447 447 ===========================
448 448
449 449 The magic function system provides a series of functions which allow you to
450 450 control the behavior of IPython itself, plus a lot of system-type
451 451 features. All these functions are prefixed with a % character, but parameters
452 452 are given without parentheses or quotes.
453 453
454 454 NOTE: If you have 'automagic' enabled (via the command line option or with the
455 455 %automagic function), you don't need to type in the % explicitly. By default,
456 456 IPython ships with automagic on, so you should only rarely need the % escape.
457 457
458 458 Example: typing '%cd mydir' (without the quotes) changes you working directory
459 459 to 'mydir', if it exists.
460 460
461 461 You can define your own magic functions to extend the system. See the supplied
462 462 ipythonrc and example-magic.py files for details (in your ipython
463 463 configuration directory, typically $HOME/.ipython/).
464 464
465 465 You can also define your own aliased names for magic functions. In your
466 466 ipythonrc file, placing a line like:
467 467
468 468 execute __IPYTHON__.magic_pf = __IPYTHON__.magic_profile
469 469
470 470 will define %pf as a new name for %profile.
471 471
472 472 You can also call magics in code using the ipmagic() function, which IPython
473 473 automatically adds to the builtin namespace. Type 'ipmagic?' for details.
474 474
475 475 For a list of the available magic functions, use %lsmagic. For a description
476 476 of any of them, type %magic_name?, e.g. '%cd?'.
477 477
478 478 Currently the magic system has the following functions:\n"""
479 479
480 480 mesc = self.shell.ESC_MAGIC
481 481 outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
482 482 "\n\n%s%s\n\n%s" % (outmsg,
483 483 magic_docs,mesc,mesc,
484 484 (' '+mesc).join(self.lsmagic()),
485 485 Magic.auto_status[self.shell.rc.automagic] ) )
486 486
487 487 page(outmsg,screen_lines=self.shell.rc.screen_length)
488 488
489 489
490 490 def magic_autoindent(self, parameter_s = ''):
491 491 """Toggle autoindent on/off (if available)."""
492 492
493 493 self.shell.set_autoindent()
494 494 print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
495 495
496 496
497 497 def magic_automagic(self, parameter_s = ''):
498 498 """Make magic functions callable without having to type the initial %.
499 499
500 500 Without argumentsl toggles on/off (when off, you must call it as
501 501 %automagic, of course). With arguments it sets the value, and you can
502 502 use any of (case insensitive):
503 503
504 504 - on,1,True: to activate
505 505
506 506 - off,0,False: to deactivate.
507 507
508 508 Note that magic functions have lowest priority, so if there's a
509 509 variable whose name collides with that of a magic fn, automagic won't
510 510 work for that function (you get the variable instead). However, if you
511 511 delete the variable (del var), the previously shadowed magic function
512 512 becomes visible to automagic again."""
513 513
514 514 rc = self.shell.rc
515 515 arg = parameter_s.lower()
516 516 if parameter_s in ('on','1','true'):
517 517 rc.automagic = True
518 518 elif parameter_s in ('off','0','false'):
519 519 rc.automagic = False
520 520 else:
521 521 rc.automagic = not rc.automagic
522 522 print '\n' + Magic.auto_status[rc.automagic]
523 523
524 524 @testdec.skip_doctest
525 525 def magic_autocall(self, parameter_s = ''):
526 526 """Make functions callable without having to type parentheses.
527 527
528 528 Usage:
529 529
530 530 %autocall [mode]
531 531
532 532 The mode can be one of: 0->Off, 1->Smart, 2->Full. If not given, the
533 533 value is toggled on and off (remembering the previous state).
534 534
535 535 In more detail, these values mean:
536 536
537 537 0 -> fully disabled
538 538
539 539 1 -> active, but do not apply if there are no arguments on the line.
540 540
541 541 In this mode, you get:
542 542
543 543 In [1]: callable
544 544 Out[1]: <built-in function callable>
545 545
546 546 In [2]: callable 'hello'
547 547 ------> callable('hello')
548 548 Out[2]: False
549 549
550 550 2 -> Active always. Even if no arguments are present, the callable
551 551 object is called:
552 552
553 553 In [2]: float
554 554 ------> float()
555 555 Out[2]: 0.0
556 556
557 557 Note that even with autocall off, you can still use '/' at the start of
558 558 a line to treat the first argument on the command line as a function
559 559 and add parentheses to it:
560 560
561 561 In [8]: /str 43
562 562 ------> str(43)
563 563 Out[8]: '43'
564 564
565 565 # all-random (note for auto-testing)
566 566 """
567 567
568 568 rc = self.shell.rc
569 569
570 570 if parameter_s:
571 571 arg = int(parameter_s)
572 572 else:
573 573 arg = 'toggle'
574 574
575 575 if not arg in (0,1,2,'toggle'):
576 576 error('Valid modes: (0->Off, 1->Smart, 2->Full')
577 577 return
578 578
579 579 if arg in (0,1,2):
580 580 rc.autocall = arg
581 581 else: # toggle
582 582 if rc.autocall:
583 583 self._magic_state.autocall_save = rc.autocall
584 584 rc.autocall = 0
585 585 else:
586 586 try:
587 587 rc.autocall = self._magic_state.autocall_save
588 588 except AttributeError:
589 589 rc.autocall = self._magic_state.autocall_save = 1
590 590
591 591 print "Automatic calling is:",['OFF','Smart','Full'][rc.autocall]
592 592
593 593 def magic_system_verbose(self, parameter_s = ''):
594 594 """Set verbose printing of system calls.
595 595
596 596 If called without an argument, act as a toggle"""
597 597
598 598 if parameter_s:
599 599 val = bool(eval(parameter_s))
600 600 else:
601 601 val = None
602 602
603 603 self.shell.rc_set_toggle('system_verbose',val)
604 604 print "System verbose printing is:",\
605 605 ['OFF','ON'][self.shell.rc.system_verbose]
606 606
607 607
608 608 def magic_page(self, parameter_s=''):
609 609 """Pretty print the object and display it through a pager.
610 610
611 611 %page [options] OBJECT
612 612
613 613 If no object is given, use _ (last output).
614 614
615 615 Options:
616 616
617 617 -r: page str(object), don't pretty-print it."""
618 618
619 619 # After a function contributed by Olivier Aubert, slightly modified.
620 620
621 621 # Process options/args
622 622 opts,args = self.parse_options(parameter_s,'r')
623 623 raw = 'r' in opts
624 624
625 625 oname = args and args or '_'
626 626 info = self._ofind(oname)
627 627 if info['found']:
628 628 txt = (raw and str or pformat)( info['obj'] )
629 629 page(txt)
630 630 else:
631 631 print 'Object `%s` not found' % oname
632 632
633 633 def magic_profile(self, parameter_s=''):
634 634 """Print your currently active IPyhton profile."""
635 635 if self.shell.rc.profile:
636 636 printpl('Current IPython profile: $self.shell.rc.profile.')
637 637 else:
638 638 print 'No profile active.'
639 639
640 640 def magic_pinfo(self, parameter_s='', namespaces=None):
641 641 """Provide detailed information about an object.
642 642
643 643 '%pinfo object' is just a synonym for object? or ?object."""
644 644
645 645 #print 'pinfo par: <%s>' % parameter_s # dbg
646 646
647 647
648 648 # detail_level: 0 -> obj? , 1 -> obj??
649 649 detail_level = 0
650 650 # We need to detect if we got called as 'pinfo pinfo foo', which can
651 651 # happen if the user types 'pinfo foo?' at the cmd line.
652 652 pinfo,qmark1,oname,qmark2 = \
653 653 re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
654 654 if pinfo or qmark1 or qmark2:
655 655 detail_level = 1
656 656 if "*" in oname:
657 657 self.magic_psearch(oname)
658 658 else:
659 659 self._inspect('pinfo', oname, detail_level=detail_level,
660 660 namespaces=namespaces)
661 661
662 662 def magic_pdef(self, parameter_s='', namespaces=None):
663 663 """Print the definition header for any callable object.
664 664
665 665 If the object is a class, print the constructor information."""
666 666 self._inspect('pdef',parameter_s, namespaces)
667 667
668 668 def magic_pdoc(self, parameter_s='', namespaces=None):
669 669 """Print the docstring for an object.
670 670
671 671 If the given object is a class, it will print both the class and the
672 672 constructor docstrings."""
673 673 self._inspect('pdoc',parameter_s, namespaces)
674 674
675 675 def magic_psource(self, parameter_s='', namespaces=None):
676 676 """Print (or run through pager) the source code for an object."""
677 677 self._inspect('psource',parameter_s, namespaces)
678 678
679 679 def magic_pfile(self, parameter_s=''):
680 680 """Print (or run through pager) the file where an object is defined.
681 681
682 682 The file opens at the line where the object definition begins. IPython
683 683 will honor the environment variable PAGER if set, and otherwise will
684 684 do its best to print the file in a convenient form.
685 685
686 686 If the given argument is not an object currently defined, IPython will
687 687 try to interpret it as a filename (automatically adding a .py extension
688 688 if needed). You can thus use %pfile as a syntax highlighting code
689 689 viewer."""
690 690
691 691 # first interpret argument as an object name
692 692 out = self._inspect('pfile',parameter_s)
693 693 # if not, try the input as a filename
694 694 if out == 'not found':
695 695 try:
696 696 filename = get_py_filename(parameter_s)
697 697 except IOError,msg:
698 698 print msg
699 699 return
700 700 page(self.shell.inspector.format(file(filename).read()))
701 701
702 702 def _inspect(self,meth,oname,namespaces=None,**kw):
703 703 """Generic interface to the inspector system.
704 704
705 705 This function is meant to be called by pdef, pdoc & friends."""
706 706
707 707 #oname = oname.strip()
708 708 #print '1- oname: <%r>' % oname # dbg
709 709 try:
710 710 oname = oname.strip().encode('ascii')
711 711 #print '2- oname: <%r>' % oname # dbg
712 712 except UnicodeEncodeError:
713 713 print 'Python identifiers can only contain ascii characters.'
714 714 return 'not found'
715 715
716 716 info = Struct(self._ofind(oname, namespaces))
717 717
718 718 if info.found:
719 719 try:
720 720 IPython.generics.inspect_object(info.obj)
721 721 return
722 722 except IPython.ipapi.TryNext:
723 723 pass
724 724 # Get the docstring of the class property if it exists.
725 725 path = oname.split('.')
726 726 root = '.'.join(path[:-1])
727 727 if info.parent is not None:
728 728 try:
729 729 target = getattr(info.parent, '__class__')
730 730 # The object belongs to a class instance.
731 731 try:
732 732 target = getattr(target, path[-1])
733 733 # The class defines the object.
734 734 if isinstance(target, property):
735 735 oname = root + '.__class__.' + path[-1]
736 736 info = Struct(self._ofind(oname))
737 737 except AttributeError: pass
738 738 except AttributeError: pass
739 739
740 740 pmethod = getattr(self.shell.inspector,meth)
741 741 formatter = info.ismagic and self.format_screen or None
742 742 if meth == 'pdoc':
743 743 pmethod(info.obj,oname,formatter)
744 744 elif meth == 'pinfo':
745 745 pmethod(info.obj,oname,formatter,info,**kw)
746 746 else:
747 747 pmethod(info.obj,oname)
748 748 else:
749 749 print 'Object `%s` not found.' % oname
750 750 return 'not found' # so callers can take other action
751 751
752 752 def magic_psearch(self, parameter_s=''):
753 753 """Search for object in namespaces by wildcard.
754 754
755 755 %psearch [options] PATTERN [OBJECT TYPE]
756 756
757 757 Note: ? can be used as a synonym for %psearch, at the beginning or at
758 758 the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the
759 759 rest of the command line must be unchanged (options come first), so
760 760 for example the following forms are equivalent
761 761
762 762 %psearch -i a* function
763 763 -i a* function?
764 764 ?-i a* function
765 765
766 766 Arguments:
767 767
768 768 PATTERN
769 769
770 770 where PATTERN is a string containing * as a wildcard similar to its
771 771 use in a shell. The pattern is matched in all namespaces on the
772 772 search path. By default objects starting with a single _ are not
773 773 matched, many IPython generated objects have a single
774 774 underscore. The default is case insensitive matching. Matching is
775 775 also done on the attributes of objects and not only on the objects
776 776 in a module.
777 777
778 778 [OBJECT TYPE]
779 779
780 780 Is the name of a python type from the types module. The name is
781 781 given in lowercase without the ending type, ex. StringType is
782 782 written string. By adding a type here only objects matching the
783 783 given type are matched. Using all here makes the pattern match all
784 784 types (this is the default).
785 785
786 786 Options:
787 787
788 788 -a: makes the pattern match even objects whose names start with a
789 789 single underscore. These names are normally ommitted from the
790 790 search.
791 791
792 792 -i/-c: make the pattern case insensitive/sensitive. If neither of
793 793 these options is given, the default is read from your ipythonrc
794 794 file. The option name which sets this value is
795 795 'wildcards_case_sensitive'. If this option is not specified in your
796 796 ipythonrc file, IPython's internal default is to do a case sensitive
797 797 search.
798 798
799 799 -e/-s NAMESPACE: exclude/search a given namespace. The pattern you
800 800 specifiy can be searched in any of the following namespaces:
801 801 'builtin', 'user', 'user_global','internal', 'alias', where
802 802 'builtin' and 'user' are the search defaults. Note that you should
803 803 not use quotes when specifying namespaces.
804 804
805 805 'Builtin' contains the python module builtin, 'user' contains all
806 806 user data, 'alias' only contain the shell aliases and no python
807 807 objects, 'internal' contains objects used by IPython. The
808 808 'user_global' namespace is only used by embedded IPython instances,
809 809 and it contains module-level globals. You can add namespaces to the
810 810 search with -s or exclude them with -e (these options can be given
811 811 more than once).
812 812
813 813 Examples:
814 814
815 815 %psearch a* -> objects beginning with an a
816 816 %psearch -e builtin a* -> objects NOT in the builtin space starting in a
817 817 %psearch a* function -> all functions beginning with an a
818 818 %psearch re.e* -> objects beginning with an e in module re
819 819 %psearch r*.e* -> objects that start with e in modules starting in r
820 820 %psearch r*.* string -> all strings in modules beginning with r
821 821
822 822 Case sensitve search:
823 823
824 824 %psearch -c a* list all object beginning with lower case a
825 825
826 826 Show objects beginning with a single _:
827 827
828 828 %psearch -a _* list objects beginning with a single underscore"""
829 829 try:
830 830 parameter_s = parameter_s.encode('ascii')
831 831 except UnicodeEncodeError:
832 832 print 'Python identifiers can only contain ascii characters.'
833 833 return
834 834
835 835 # default namespaces to be searched
836 836 def_search = ['user','builtin']
837 837
838 838 # Process options/args
839 839 opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
840 840 opt = opts.get
841 841 shell = self.shell
842 842 psearch = shell.inspector.psearch
843 843
844 844 # select case options
845 845 if opts.has_key('i'):
846 846 ignore_case = True
847 847 elif opts.has_key('c'):
848 848 ignore_case = False
849 849 else:
850 850 ignore_case = not shell.rc.wildcards_case_sensitive
851 851
852 852 # Build list of namespaces to search from user options
853 853 def_search.extend(opt('s',[]))
854 854 ns_exclude = ns_exclude=opt('e',[])
855 855 ns_search = [nm for nm in def_search if nm not in ns_exclude]
856 856
857 857 # Call the actual search
858 858 try:
859 859 psearch(args,shell.ns_table,ns_search,
860 860 show_all=opt('a'),ignore_case=ignore_case)
861 861 except:
862 862 shell.showtraceback()
863 863
864 864 def magic_who_ls(self, parameter_s=''):
865 865 """Return a sorted list of all interactive variables.
866 866
867 867 If arguments are given, only variables of types matching these
868 868 arguments are returned."""
869 869
870 870 user_ns = self.shell.user_ns
871 871 internal_ns = self.shell.internal_ns
872 872 user_config_ns = self.shell.user_config_ns
873 873 out = []
874 874 typelist = parameter_s.split()
875 875
876 876 for i in user_ns:
877 877 if not (i.startswith('_') or i.startswith('_i')) \
878 878 and not (i in internal_ns or i in user_config_ns):
879 879 if typelist:
880 880 if type(user_ns[i]).__name__ in typelist:
881 881 out.append(i)
882 882 else:
883 883 out.append(i)
884 884 out.sort()
885 885 return out
886 886
887 887 def magic_who(self, parameter_s=''):
888 888 """Print all interactive variables, with some minimal formatting.
889 889
890 890 If any arguments are given, only variables whose type matches one of
891 891 these are printed. For example:
892 892
893 893 %who function str
894 894
895 895 will only list functions and strings, excluding all other types of
896 896 variables. To find the proper type names, simply use type(var) at a
897 897 command line to see how python prints type names. For example:
898 898
899 899 In [1]: type('hello')\\
900 900 Out[1]: <type 'str'>
901 901
902 902 indicates that the type name for strings is 'str'.
903 903
904 904 %who always excludes executed names loaded through your configuration
905 905 file and things which are internal to IPython.
906 906
907 907 This is deliberate, as typically you may load many modules and the
908 908 purpose of %who is to show you only what you've manually defined."""
909 909
910 910 varlist = self.magic_who_ls(parameter_s)
911 911 if not varlist:
912 912 if parameter_s:
913 913 print 'No variables match your requested type.'
914 914 else:
915 915 print 'Interactive namespace is empty.'
916 916 return
917 917
918 918 # if we have variables, move on...
919 919 count = 0
920 920 for i in varlist:
921 921 print i+'\t',
922 922 count += 1
923 923 if count > 8:
924 924 count = 0
925 925 print
926 926 print
927 927
928 928 def magic_whos(self, parameter_s=''):
929 929 """Like %who, but gives some extra information about each variable.
930 930
931 931 The same type filtering of %who can be applied here.
932 932
933 933 For all variables, the type is printed. Additionally it prints:
934 934
935 935 - For {},[],(): their length.
936 936
937 937 - For numpy and Numeric arrays, a summary with shape, number of
938 938 elements, typecode and size in memory.
939 939
940 940 - Everything else: a string representation, snipping their middle if
941 941 too long."""
942 942
943 943 varnames = self.magic_who_ls(parameter_s)
944 944 if not varnames:
945 945 if parameter_s:
946 946 print 'No variables match your requested type.'
947 947 else:
948 948 print 'Interactive namespace is empty.'
949 949 return
950 950
951 951 # if we have variables, move on...
952 952
953 953 # for these types, show len() instead of data:
954 954 seq_types = [types.DictType,types.ListType,types.TupleType]
955 955
956 956 # for numpy/Numeric arrays, display summary info
957 957 try:
958 958 import numpy
959 959 except ImportError:
960 960 ndarray_type = None
961 961 else:
962 962 ndarray_type = numpy.ndarray.__name__
963 963 try:
964 964 import Numeric
965 965 except ImportError:
966 966 array_type = None
967 967 else:
968 968 array_type = Numeric.ArrayType.__name__
969 969
970 970 # Find all variable names and types so we can figure out column sizes
971 971 def get_vars(i):
972 972 return self.shell.user_ns[i]
973 973
974 974 # some types are well known and can be shorter
975 975 abbrevs = {'IPython.macro.Macro' : 'Macro'}
976 976 def type_name(v):
977 977 tn = type(v).__name__
978 978 return abbrevs.get(tn,tn)
979 979
980 980 varlist = map(get_vars,varnames)
981 981
982 982 typelist = []
983 983 for vv in varlist:
984 984 tt = type_name(vv)
985 985
986 986 if tt=='instance':
987 987 typelist.append( abbrevs.get(str(vv.__class__),
988 988 str(vv.__class__)))
989 989 else:
990 990 typelist.append(tt)
991 991
992 992 # column labels and # of spaces as separator
993 993 varlabel = 'Variable'
994 994 typelabel = 'Type'
995 995 datalabel = 'Data/Info'
996 996 colsep = 3
997 997 # variable format strings
998 998 vformat = "$vname.ljust(varwidth)$vtype.ljust(typewidth)"
999 999 vfmt_short = '$vstr[:25]<...>$vstr[-25:]'
1000 1000 aformat = "%s: %s elems, type `%s`, %s bytes"
1001 1001 # find the size of the columns to format the output nicely
1002 1002 varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
1003 1003 typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
1004 1004 # table header
1005 1005 print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
1006 1006 ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
1007 1007 # and the table itself
1008 1008 kb = 1024
1009 1009 Mb = 1048576 # kb**2
1010 1010 for vname,var,vtype in zip(varnames,varlist,typelist):
1011 1011 print itpl(vformat),
1012 1012 if vtype in seq_types:
1013 1013 print len(var)
1014 1014 elif vtype in [array_type,ndarray_type]:
1015 1015 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
1016 1016 if vtype==ndarray_type:
1017 1017 # numpy
1018 1018 vsize = var.size
1019 1019 vbytes = vsize*var.itemsize
1020 1020 vdtype = var.dtype
1021 1021 else:
1022 1022 # Numeric
1023 1023 vsize = Numeric.size(var)
1024 1024 vbytes = vsize*var.itemsize()
1025 1025 vdtype = var.typecode()
1026 1026
1027 1027 if vbytes < 100000:
1028 1028 print aformat % (vshape,vsize,vdtype,vbytes)
1029 1029 else:
1030 1030 print aformat % (vshape,vsize,vdtype,vbytes),
1031 1031 if vbytes < Mb:
1032 1032 print '(%s kb)' % (vbytes/kb,)
1033 1033 else:
1034 1034 print '(%s Mb)' % (vbytes/Mb,)
1035 1035 else:
1036 1036 try:
1037 1037 vstr = str(var)
1038 1038 except UnicodeEncodeError:
1039 1039 vstr = unicode(var).encode(sys.getdefaultencoding(),
1040 1040 'backslashreplace')
1041 1041 vstr = vstr.replace('\n','\\n')
1042 1042 if len(vstr) < 50:
1043 1043 print vstr
1044 1044 else:
1045 1045 printpl(vfmt_short)
1046 1046
1047 1047 def magic_reset(self, parameter_s=''):
1048 1048 """Resets the namespace by removing all names defined by the user.
1049 1049
1050 1050 Input/Output history are left around in case you need them.
1051 1051
1052 1052 Parameters
1053 1053 ----------
1054 1054 -y : force reset without asking for confirmation.
1055 1055
1056 1056 Examples
1057 1057 --------
1058 1058 In [6]: a = 1
1059 1059
1060 1060 In [7]: a
1061 1061 Out[7]: 1
1062 1062
1063 1063 In [8]: 'a' in _ip.user_ns
1064 1064 Out[8]: True
1065 1065
1066 1066 In [9]: %reset -f
1067 1067
1068 1068 In [10]: 'a' in _ip.user_ns
1069 1069 Out[10]: False
1070 1070 """
1071 1071
1072 1072 if parameter_s == '-f':
1073 1073 ans = True
1074 1074 else:
1075 1075 ans = self.shell.ask_yes_no(
1076 1076 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ")
1077 1077 if not ans:
1078 1078 print 'Nothing done.'
1079 1079 return
1080 1080 user_ns = self.shell.user_ns
1081 1081 for i in self.magic_who_ls():
1082 1082 del(user_ns[i])
1083 1083
1084 1084 # Also flush the private list of module references kept for script
1085 1085 # execution protection
1086 1086 self.shell.clear_main_mod_cache()
1087 1087
1088 1088 def magic_logstart(self,parameter_s=''):
1089 1089 """Start logging anywhere in a session.
1090 1090
1091 1091 %logstart [-o|-r|-t] [log_name [log_mode]]
1092 1092
1093 1093 If no name is given, it defaults to a file named 'ipython_log.py' in your
1094 1094 current directory, in 'rotate' mode (see below).
1095 1095
1096 1096 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
1097 1097 history up to that point and then continues logging.
1098 1098
1099 1099 %logstart takes a second optional parameter: logging mode. This can be one
1100 1100 of (note that the modes are given unquoted):\\
1101 1101 append: well, that says it.\\
1102 1102 backup: rename (if exists) to name~ and start name.\\
1103 1103 global: single logfile in your home dir, appended to.\\
1104 1104 over : overwrite existing log.\\
1105 1105 rotate: create rotating logs name.1~, name.2~, etc.
1106 1106
1107 1107 Options:
1108 1108
1109 1109 -o: log also IPython's output. In this mode, all commands which
1110 1110 generate an Out[NN] prompt are recorded to the logfile, right after
1111 1111 their corresponding input line. The output lines are always
1112 1112 prepended with a '#[Out]# ' marker, so that the log remains valid
1113 1113 Python code.
1114 1114
1115 1115 Since this marker is always the same, filtering only the output from
1116 1116 a log is very easy, using for example a simple awk call:
1117 1117
1118 1118 awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
1119 1119
1120 1120 -r: log 'raw' input. Normally, IPython's logs contain the processed
1121 1121 input, so that user lines are logged in their final form, converted
1122 1122 into valid Python. For example, %Exit is logged as
1123 1123 '_ip.magic("Exit"). If the -r flag is given, all input is logged
1124 1124 exactly as typed, with no transformations applied.
1125 1125
1126 1126 -t: put timestamps before each input line logged (these are put in
1127 1127 comments)."""
1128 1128
1129 1129 opts,par = self.parse_options(parameter_s,'ort')
1130 1130 log_output = 'o' in opts
1131 1131 log_raw_input = 'r' in opts
1132 1132 timestamp = 't' in opts
1133 1133
1134 1134 rc = self.shell.rc
1135 1135 logger = self.shell.logger
1136 1136
1137 1137 # if no args are given, the defaults set in the logger constructor by
1138 1138 # ipytohn remain valid
1139 1139 if par:
1140 1140 try:
1141 1141 logfname,logmode = par.split()
1142 1142 except:
1143 1143 logfname = par
1144 1144 logmode = 'backup'
1145 1145 else:
1146 1146 logfname = logger.logfname
1147 1147 logmode = logger.logmode
1148 1148 # put logfname into rc struct as if it had been called on the command
1149 1149 # line, so it ends up saved in the log header Save it in case we need
1150 1150 # to restore it...
1151 1151 old_logfile = rc.opts.get('logfile','')
1152 1152 if logfname:
1153 1153 logfname = os.path.expanduser(logfname)
1154 1154 rc.opts.logfile = logfname
1155 1155 loghead = self.shell.loghead_tpl % (rc.opts,rc.args)
1156 1156 try:
1157 1157 started = logger.logstart(logfname,loghead,logmode,
1158 1158 log_output,timestamp,log_raw_input)
1159 1159 except:
1160 1160 rc.opts.logfile = old_logfile
1161 1161 warn("Couldn't start log: %s" % sys.exc_info()[1])
1162 1162 else:
1163 1163 # log input history up to this point, optionally interleaving
1164 1164 # output if requested
1165 1165
1166 1166 if timestamp:
1167 1167 # disable timestamping for the previous history, since we've
1168 1168 # lost those already (no time machine here).
1169 1169 logger.timestamp = False
1170 1170
1171 1171 if log_raw_input:
1172 1172 input_hist = self.shell.input_hist_raw
1173 1173 else:
1174 1174 input_hist = self.shell.input_hist
1175 1175
1176 1176 if log_output:
1177 1177 log_write = logger.log_write
1178 1178 output_hist = self.shell.output_hist
1179 1179 for n in range(1,len(input_hist)-1):
1180 1180 log_write(input_hist[n].rstrip())
1181 1181 if n in output_hist:
1182 1182 log_write(repr(output_hist[n]),'output')
1183 1183 else:
1184 1184 logger.log_write(input_hist[1:])
1185 1185 if timestamp:
1186 1186 # re-enable timestamping
1187 1187 logger.timestamp = True
1188 1188
1189 1189 print ('Activating auto-logging. '
1190 1190 'Current session state plus future input saved.')
1191 1191 logger.logstate()
1192 1192
1193 1193 def magic_logstop(self,parameter_s=''):
1194 1194 """Fully stop logging and close log file.
1195 1195
1196 1196 In order to start logging again, a new %logstart call needs to be made,
1197 1197 possibly (though not necessarily) with a new filename, mode and other
1198 1198 options."""
1199 1199 self.logger.logstop()
1200 1200
1201 1201 def magic_logoff(self,parameter_s=''):
1202 1202 """Temporarily stop logging.
1203 1203
1204 1204 You must have previously started logging."""
1205 1205 self.shell.logger.switch_log(0)
1206 1206
1207 1207 def magic_logon(self,parameter_s=''):
1208 1208 """Restart logging.
1209 1209
1210 1210 This function is for restarting logging which you've temporarily
1211 1211 stopped with %logoff. For starting logging for the first time, you
1212 1212 must use the %logstart function, which allows you to specify an
1213 1213 optional log filename."""
1214 1214
1215 1215 self.shell.logger.switch_log(1)
1216 1216
1217 1217 def magic_logstate(self,parameter_s=''):
1218 1218 """Print the status of the logging system."""
1219 1219
1220 1220 self.shell.logger.logstate()
1221 1221
1222 1222 def magic_pdb(self, parameter_s=''):
1223 1223 """Control the automatic calling of the pdb interactive debugger.
1224 1224
1225 1225 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
1226 1226 argument it works as a toggle.
1227 1227
1228 1228 When an exception is triggered, IPython can optionally call the
1229 1229 interactive pdb debugger after the traceback printout. %pdb toggles
1230 1230 this feature on and off.
1231 1231
1232 1232 The initial state of this feature is set in your ipythonrc
1233 1233 configuration file (the variable is called 'pdb').
1234 1234
1235 1235 If you want to just activate the debugger AFTER an exception has fired,
1236 1236 without having to type '%pdb on' and rerunning your code, you can use
1237 1237 the %debug magic."""
1238 1238
1239 1239 par = parameter_s.strip().lower()
1240 1240
1241 1241 if par:
1242 1242 try:
1243 1243 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
1244 1244 except KeyError:
1245 1245 print ('Incorrect argument. Use on/1, off/0, '
1246 1246 'or nothing for a toggle.')
1247 1247 return
1248 1248 else:
1249 1249 # toggle
1250 1250 new_pdb = not self.shell.call_pdb
1251 1251
1252 1252 # set on the shell
1253 1253 self.shell.call_pdb = new_pdb
1254 1254 print 'Automatic pdb calling has been turned',on_off(new_pdb)
1255 1255
1256 1256 def magic_debug(self, parameter_s=''):
1257 1257 """Activate the interactive debugger in post-mortem mode.
1258 1258
1259 1259 If an exception has just occurred, this lets you inspect its stack
1260 1260 frames interactively. Note that this will always work only on the last
1261 1261 traceback that occurred, so you must call this quickly after an
1262 1262 exception that you wish to inspect has fired, because if another one
1263 1263 occurs, it clobbers the previous one.
1264 1264
1265 1265 If you want IPython to automatically do this on every exception, see
1266 1266 the %pdb magic for more details.
1267 1267 """
1268 1268
1269 1269 self.shell.debugger(force=True)
1270 1270
1271 1271 @testdec.skip_doctest
1272 1272 def magic_prun(self, parameter_s ='',user_mode=1,
1273 1273 opts=None,arg_lst=None,prog_ns=None):
1274 1274
1275 1275 """Run a statement through the python code profiler.
1276 1276
1277 1277 Usage:
1278 1278 %prun [options] statement
1279 1279
1280 1280 The given statement (which doesn't require quote marks) is run via the
1281 1281 python profiler in a manner similar to the profile.run() function.
1282 1282 Namespaces are internally managed to work correctly; profile.run
1283 1283 cannot be used in IPython because it makes certain assumptions about
1284 1284 namespaces which do not hold under IPython.
1285 1285
1286 1286 Options:
1287 1287
1288 1288 -l <limit>: you can place restrictions on what or how much of the
1289 1289 profile gets printed. The limit value can be:
1290 1290
1291 1291 * A string: only information for function names containing this string
1292 1292 is printed.
1293 1293
1294 1294 * An integer: only these many lines are printed.
1295 1295
1296 1296 * A float (between 0 and 1): this fraction of the report is printed
1297 1297 (for example, use a limit of 0.4 to see the topmost 40% only).
1298 1298
1299 1299 You can combine several limits with repeated use of the option. For
1300 1300 example, '-l __init__ -l 5' will print only the topmost 5 lines of
1301 1301 information about class constructors.
1302 1302
1303 1303 -r: return the pstats.Stats object generated by the profiling. This
1304 1304 object has all the information about the profile in it, and you can
1305 1305 later use it for further analysis or in other functions.
1306 1306
1307 1307 -s <key>: sort profile by given key. You can provide more than one key
1308 1308 by using the option several times: '-s key1 -s key2 -s key3...'. The
1309 1309 default sorting key is 'time'.
1310 1310
1311 1311 The following is copied verbatim from the profile documentation
1312 1312 referenced below:
1313 1313
1314 1314 When more than one key is provided, additional keys are used as
1315 1315 secondary criteria when the there is equality in all keys selected
1316 1316 before them.
1317 1317
1318 1318 Abbreviations can be used for any key names, as long as the
1319 1319 abbreviation is unambiguous. The following are the keys currently
1320 1320 defined:
1321 1321
1322 1322 Valid Arg Meaning
1323 1323 "calls" call count
1324 1324 "cumulative" cumulative time
1325 1325 "file" file name
1326 1326 "module" file name
1327 1327 "pcalls" primitive call count
1328 1328 "line" line number
1329 1329 "name" function name
1330 1330 "nfl" name/file/line
1331 1331 "stdname" standard name
1332 1332 "time" internal time
1333 1333
1334 1334 Note that all sorts on statistics are in descending order (placing
1335 1335 most time consuming items first), where as name, file, and line number
1336 1336 searches are in ascending order (i.e., alphabetical). The subtle
1337 1337 distinction between "nfl" and "stdname" is that the standard name is a
1338 1338 sort of the name as printed, which means that the embedded line
1339 1339 numbers get compared in an odd way. For example, lines 3, 20, and 40
1340 1340 would (if the file names were the same) appear in the string order
1341 1341 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
1342 1342 line numbers. In fact, sort_stats("nfl") is the same as
1343 1343 sort_stats("name", "file", "line").
1344 1344
1345 1345 -T <filename>: save profile results as shown on screen to a text
1346 1346 file. The profile is still shown on screen.
1347 1347
1348 1348 -D <filename>: save (via dump_stats) profile statistics to given
1349 1349 filename. This data is in a format understod by the pstats module, and
1350 1350 is generated by a call to the dump_stats() method of profile
1351 1351 objects. The profile is still shown on screen.
1352 1352
1353 1353 If you want to run complete programs under the profiler's control, use
1354 1354 '%run -p [prof_opts] filename.py [args to program]' where prof_opts
1355 1355 contains profiler specific options as described here.
1356 1356
1357 1357 You can read the complete documentation for the profile module with::
1358 1358
1359 1359 In [1]: import profile; profile.help()
1360 1360 """
1361 1361
1362 1362 opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
1363 1363 # protect user quote marks
1364 1364 parameter_s = parameter_s.replace('"',r'\"').replace("'",r"\'")
1365 1365
1366 1366 if user_mode: # regular user call
1367 1367 opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:',
1368 1368 list_all=1)
1369 1369 namespace = self.shell.user_ns
1370 1370 else: # called to run a program by %run -p
1371 1371 try:
1372 1372 filename = get_py_filename(arg_lst[0])
1373 1373 except IOError,msg:
1374 1374 error(msg)
1375 1375 return
1376 1376
1377 1377 arg_str = 'execfile(filename,prog_ns)'
1378 1378 namespace = locals()
1379 1379
1380 1380 opts.merge(opts_def)
1381 1381
1382 1382 prof = profile.Profile()
1383 1383 try:
1384 1384 prof = prof.runctx(arg_str,namespace,namespace)
1385 1385 sys_exit = ''
1386 1386 except SystemExit:
1387 1387 sys_exit = """*** SystemExit exception caught in code being profiled."""
1388 1388
1389 1389 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
1390 1390
1391 1391 lims = opts.l
1392 1392 if lims:
1393 1393 lims = [] # rebuild lims with ints/floats/strings
1394 1394 for lim in opts.l:
1395 1395 try:
1396 1396 lims.append(int(lim))
1397 1397 except ValueError:
1398 1398 try:
1399 1399 lims.append(float(lim))
1400 1400 except ValueError:
1401 1401 lims.append(lim)
1402 1402
1403 1403 # Trap output.
1404 1404 stdout_trap = StringIO()
1405 1405
1406 1406 if hasattr(stats,'stream'):
1407 1407 # In newer versions of python, the stats object has a 'stream'
1408 1408 # attribute to write into.
1409 1409 stats.stream = stdout_trap
1410 1410 stats.print_stats(*lims)
1411 1411 else:
1412 1412 # For older versions, we manually redirect stdout during printing
1413 1413 sys_stdout = sys.stdout
1414 1414 try:
1415 1415 sys.stdout = stdout_trap
1416 1416 stats.print_stats(*lims)
1417 1417 finally:
1418 1418 sys.stdout = sys_stdout
1419 1419
1420 1420 output = stdout_trap.getvalue()
1421 1421 output = output.rstrip()
1422 1422
1423 1423 page(output,screen_lines=self.shell.rc.screen_length)
1424 1424 print sys_exit,
1425 1425
1426 1426 dump_file = opts.D[0]
1427 1427 text_file = opts.T[0]
1428 1428 if dump_file:
1429 1429 prof.dump_stats(dump_file)
1430 1430 print '\n*** Profile stats marshalled to file',\
1431 1431 `dump_file`+'.',sys_exit
1432 1432 if text_file:
1433 1433 pfile = file(text_file,'w')
1434 1434 pfile.write(output)
1435 1435 pfile.close()
1436 1436 print '\n*** Profile printout saved to text file',\
1437 1437 `text_file`+'.',sys_exit
1438 1438
1439 1439 if opts.has_key('r'):
1440 1440 return stats
1441 1441 else:
1442 1442 return None
1443 1443
1444 1444 @testdec.skip_doctest
1445 1445 def magic_run(self, parameter_s ='',runner=None,
1446 1446 file_finder=get_py_filename):
1447 1447 """Run the named file inside IPython as a program.
1448 1448
1449 1449 Usage:\\
1450 1450 %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
1451 1451
1452 1452 Parameters after the filename are passed as command-line arguments to
1453 1453 the program (put in sys.argv). Then, control returns to IPython's
1454 1454 prompt.
1455 1455
1456 1456 This is similar to running at a system prompt:\\
1457 1457 $ python file args\\
1458 1458 but with the advantage of giving you IPython's tracebacks, and of
1459 1459 loading all variables into your interactive namespace for further use
1460 1460 (unless -p is used, see below).
1461 1461
1462 1462 The file is executed in a namespace initially consisting only of
1463 1463 __name__=='__main__' and sys.argv constructed as indicated. It thus
1464 1464 sees its environment as if it were being run as a stand-alone program
1465 1465 (except for sharing global objects such as previously imported
1466 1466 modules). But after execution, the IPython interactive namespace gets
1467 1467 updated with all variables defined in the program (except for __name__
1468 1468 and sys.argv). This allows for very convenient loading of code for
1469 1469 interactive work, while giving each program a 'clean sheet' to run in.
1470 1470
1471 1471 Options:
1472 1472
1473 1473 -n: __name__ is NOT set to '__main__', but to the running file's name
1474 1474 without extension (as python does under import). This allows running
1475 1475 scripts and reloading the definitions in them without calling code
1476 1476 protected by an ' if __name__ == "__main__" ' clause.
1477 1477
1478 1478 -i: run the file in IPython's namespace instead of an empty one. This
1479 1479 is useful if you are experimenting with code written in a text editor
1480 1480 which depends on variables defined interactively.
1481 1481
1482 1482 -e: ignore sys.exit() calls or SystemExit exceptions in the script
1483 1483 being run. This is particularly useful if IPython is being used to
1484 1484 run unittests, which always exit with a sys.exit() call. In such
1485 1485 cases you are interested in the output of the test results, not in
1486 1486 seeing a traceback of the unittest module.
1487 1487
1488 1488 -t: print timing information at the end of the run. IPython will give
1489 1489 you an estimated CPU time consumption for your script, which under
1490 1490 Unix uses the resource module to avoid the wraparound problems of
1491 1491 time.clock(). Under Unix, an estimate of time spent on system tasks
1492 1492 is also given (for Windows platforms this is reported as 0.0).
1493 1493
1494 1494 If -t is given, an additional -N<N> option can be given, where <N>
1495 1495 must be an integer indicating how many times you want the script to
1496 1496 run. The final timing report will include total and per run results.
1497 1497
1498 1498 For example (testing the script uniq_stable.py):
1499 1499
1500 1500 In [1]: run -t uniq_stable
1501 1501
1502 1502 IPython CPU timings (estimated):\\
1503 1503 User : 0.19597 s.\\
1504 1504 System: 0.0 s.\\
1505 1505
1506 1506 In [2]: run -t -N5 uniq_stable
1507 1507
1508 1508 IPython CPU timings (estimated):\\
1509 1509 Total runs performed: 5\\
1510 1510 Times : Total Per run\\
1511 1511 User : 0.910862 s, 0.1821724 s.\\
1512 1512 System: 0.0 s, 0.0 s.
1513 1513
1514 1514 -d: run your program under the control of pdb, the Python debugger.
1515 1515 This allows you to execute your program step by step, watch variables,
1516 1516 etc. Internally, what IPython does is similar to calling:
1517 1517
1518 1518 pdb.run('execfile("YOURFILENAME")')
1519 1519
1520 1520 with a breakpoint set on line 1 of your file. You can change the line
1521 1521 number for this automatic breakpoint to be <N> by using the -bN option
1522 1522 (where N must be an integer). For example:
1523 1523
1524 1524 %run -d -b40 myscript
1525 1525
1526 1526 will set the first breakpoint at line 40 in myscript.py. Note that
1527 1527 the first breakpoint must be set on a line which actually does
1528 1528 something (not a comment or docstring) for it to stop execution.
1529 1529
1530 1530 When the pdb debugger starts, you will see a (Pdb) prompt. You must
1531 1531 first enter 'c' (without qoutes) to start execution up to the first
1532 1532 breakpoint.
1533 1533
1534 1534 Entering 'help' gives information about the use of the debugger. You
1535 1535 can easily see pdb's full documentation with "import pdb;pdb.help()"
1536 1536 at a prompt.
1537 1537
1538 1538 -p: run program under the control of the Python profiler module (which
1539 1539 prints a detailed report of execution times, function calls, etc).
1540 1540
1541 1541 You can pass other options after -p which affect the behavior of the
1542 1542 profiler itself. See the docs for %prun for details.
1543 1543
1544 1544 In this mode, the program's variables do NOT propagate back to the
1545 1545 IPython interactive namespace (because they remain in the namespace
1546 1546 where the profiler executes them).
1547 1547
1548 1548 Internally this triggers a call to %prun, see its documentation for
1549 1549 details on the options available specifically for profiling.
1550 1550
1551 1551 There is one special usage for which the text above doesn't apply:
1552 1552 if the filename ends with .ipy, the file is run as ipython script,
1553 1553 just as if the commands were written on IPython prompt.
1554 1554 """
1555 1555
1556 1556 # get arguments and set sys.argv for program to be run.
1557 1557 opts,arg_lst = self.parse_options(parameter_s,'nidtN:b:pD:l:rs:T:e',
1558 1558 mode='list',list_all=1)
1559 1559
1560 1560 try:
1561 1561 filename = file_finder(arg_lst[0])
1562 1562 except IndexError:
1563 1563 warn('you must provide at least a filename.')
1564 1564 print '\n%run:\n',OInspect.getdoc(self.magic_run)
1565 1565 return
1566 1566 except IOError,msg:
1567 1567 error(msg)
1568 1568 return
1569 1569
1570 1570 if filename.lower().endswith('.ipy'):
1571 1571 self.api.runlines(open(filename).read())
1572 1572 return
1573 1573
1574 1574 # Control the response to exit() calls made by the script being run
1575 1575 exit_ignore = opts.has_key('e')
1576 1576
1577 1577 # Make sure that the running script gets a proper sys.argv as if it
1578 1578 # were run from a system shell.
1579 1579 save_argv = sys.argv # save it for later restoring
1580 1580 sys.argv = [filename]+ arg_lst[1:] # put in the proper filename
1581 1581
1582 1582 if opts.has_key('i'):
1583 1583 # Run in user's interactive namespace
1584 1584 prog_ns = self.shell.user_ns
1585 1585 __name__save = self.shell.user_ns['__name__']
1586 1586 prog_ns['__name__'] = '__main__'
1587 1587 main_mod = self.shell.new_main_mod(prog_ns)
1588 1588 else:
1589 1589 # Run in a fresh, empty namespace
1590 1590 if opts.has_key('n'):
1591 1591 name = os.path.splitext(os.path.basename(filename))[0]
1592 1592 else:
1593 1593 name = '__main__'
1594 1594
1595 1595 main_mod = self.shell.new_main_mod()
1596 1596 prog_ns = main_mod.__dict__
1597 1597 prog_ns['__name__'] = name
1598 1598
1599 1599 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
1600 1600 # set the __file__ global in the script's namespace
1601 1601 prog_ns['__file__'] = filename
1602 1602
1603 1603 # pickle fix. See iplib for an explanation. But we need to make sure
1604 1604 # that, if we overwrite __main__, we replace it at the end
1605 1605 main_mod_name = prog_ns['__name__']
1606 1606
1607 1607 if main_mod_name == '__main__':
1608 1608 restore_main = sys.modules['__main__']
1609 1609 else:
1610 1610 restore_main = False
1611 1611
1612 1612 # This needs to be undone at the end to prevent holding references to
1613 1613 # every single object ever created.
1614 1614 sys.modules[main_mod_name] = main_mod
1615 1615
1616 1616 stats = None
1617 1617 try:
1618 1618 self.shell.savehist()
1619 1619
1620 1620 if opts.has_key('p'):
1621 1621 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
1622 1622 else:
1623 1623 if opts.has_key('d'):
1624 1624 deb = Debugger.Pdb(self.shell.rc.colors)
1625 1625 # reset Breakpoint state, which is moronically kept
1626 1626 # in a class
1627 1627 bdb.Breakpoint.next = 1
1628 1628 bdb.Breakpoint.bplist = {}
1629 1629 bdb.Breakpoint.bpbynumber = [None]
1630 1630 # Set an initial breakpoint to stop execution
1631 1631 maxtries = 10
1632 1632 bp = int(opts.get('b',[1])[0])
1633 1633 checkline = deb.checkline(filename,bp)
1634 1634 if not checkline:
1635 1635 for bp in range(bp+1,bp+maxtries+1):
1636 1636 if deb.checkline(filename,bp):
1637 1637 break
1638 1638 else:
1639 1639 msg = ("\nI failed to find a valid line to set "
1640 1640 "a breakpoint\n"
1641 1641 "after trying up to line: %s.\n"
1642 1642 "Please set a valid breakpoint manually "
1643 1643 "with the -b option." % bp)
1644 1644 error(msg)
1645 1645 return
1646 1646 # if we find a good linenumber, set the breakpoint
1647 1647 deb.do_break('%s:%s' % (filename,bp))
1648 1648 # Start file run
1649 1649 print "NOTE: Enter 'c' at the",
1650 1650 print "%s prompt to start your script." % deb.prompt
1651 1651 try:
1652 1652 deb.run('execfile("%s")' % filename,prog_ns)
1653 1653
1654 1654 except:
1655 1655 etype, value, tb = sys.exc_info()
1656 1656 # Skip three frames in the traceback: the %run one,
1657 1657 # one inside bdb.py, and the command-line typed by the
1658 1658 # user (run by exec in pdb itself).
1659 1659 self.shell.InteractiveTB(etype,value,tb,tb_offset=3)
1660 1660 else:
1661 1661 if runner is None:
1662 1662 runner = self.shell.safe_execfile
1663 1663 if opts.has_key('t'):
1664 1664 # timed execution
1665 1665 try:
1666 1666 nruns = int(opts['N'][0])
1667 1667 if nruns < 1:
1668 1668 error('Number of runs must be >=1')
1669 1669 return
1670 1670 except (KeyError):
1671 1671 nruns = 1
1672 1672 if nruns == 1:
1673 1673 t0 = clock2()
1674 1674 runner(filename,prog_ns,prog_ns,
1675 1675 exit_ignore=exit_ignore)
1676 1676 t1 = clock2()
1677 1677 t_usr = t1[0]-t0[0]
1678 1678 t_sys = t1[1]-t0[1]
1679 1679 print "\nIPython CPU timings (estimated):"
1680 1680 print " User : %10s s." % t_usr
1681 1681 print " System: %10s s." % t_sys
1682 1682 else:
1683 1683 runs = range(nruns)
1684 1684 t0 = clock2()
1685 1685 for nr in runs:
1686 1686 runner(filename,prog_ns,prog_ns,
1687 1687 exit_ignore=exit_ignore)
1688 1688 t1 = clock2()
1689 1689 t_usr = t1[0]-t0[0]
1690 1690 t_sys = t1[1]-t0[1]
1691 1691 print "\nIPython CPU timings (estimated):"
1692 1692 print "Total runs performed:",nruns
1693 1693 print " Times : %10s %10s" % ('Total','Per run')
1694 1694 print " User : %10s s, %10s s." % (t_usr,t_usr/nruns)
1695 1695 print " System: %10s s, %10s s." % (t_sys,t_sys/nruns)
1696 1696
1697 1697 else:
1698 1698 # regular execution
1699 1699 runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore)
1700 1700
1701 1701 if opts.has_key('i'):
1702 1702 self.shell.user_ns['__name__'] = __name__save
1703 1703 else:
1704 1704 # The shell MUST hold a reference to prog_ns so after %run
1705 1705 # exits, the python deletion mechanism doesn't zero it out
1706 1706 # (leaving dangling references).
1707 1707 self.shell.cache_main_mod(prog_ns,filename)
1708 1708 # update IPython interactive namespace
1709 1709 del prog_ns['__name__']
1710 1710 self.shell.user_ns.update(prog_ns)
1711 1711 finally:
1712 # It's a bit of a mystery why, but __builtins__ can change from
1713 # being a module to becoming a dict missing some key data after
1714 # %run. As best I can see, this is NOT something IPython is doing
1715 # at all, and similar problems have been reported before:
1716 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
1717 # Since this seems to be done by the interpreter itself, the best
1718 # we can do is to at least restore __builtins__ for the user on
1719 # exit.
1720 self.shell.user_ns['__builtins__'] = __builtin__
1721
1712 1722 # Ensure key global structures are restored
1713 1723 sys.argv = save_argv
1714 1724 if restore_main:
1715 1725 sys.modules['__main__'] = restore_main
1716 1726 else:
1717 1727 # Remove from sys.modules the reference to main_mod we'd
1718 1728 # added. Otherwise it will trap references to objects
1719 1729 # contained therein.
1720 1730 del sys.modules[main_mod_name]
1721 1731
1722 1732 self.shell.reloadhist()
1723 1733
1724 1734 return stats
1725 1735
1726 1736 def magic_runlog(self, parameter_s =''):
1727 1737 """Run files as logs.
1728 1738
1729 1739 Usage:\\
1730 1740 %runlog file1 file2 ...
1731 1741
1732 1742 Run the named files (treating them as log files) in sequence inside
1733 1743 the interpreter, and return to the prompt. This is much slower than
1734 1744 %run because each line is executed in a try/except block, but it
1735 1745 allows running files with syntax errors in them.
1736 1746
1737 1747 Normally IPython will guess when a file is one of its own logfiles, so
1738 1748 you can typically use %run even for logs. This shorthand allows you to
1739 1749 force any file to be treated as a log file."""
1740 1750
1741 1751 for f in parameter_s.split():
1742 1752 self.shell.safe_execfile(f,self.shell.user_ns,
1743 1753 self.shell.user_ns,islog=1)
1744 1754
1745 1755 @testdec.skip_doctest
1746 1756 def magic_timeit(self, parameter_s =''):
1747 1757 """Time execution of a Python statement or expression
1748 1758
1749 1759 Usage:\\
1750 1760 %timeit [-n<N> -r<R> [-t|-c]] statement
1751 1761
1752 1762 Time execution of a Python statement or expression using the timeit
1753 1763 module.
1754 1764
1755 1765 Options:
1756 1766 -n<N>: execute the given statement <N> times in a loop. If this value
1757 1767 is not given, a fitting value is chosen.
1758 1768
1759 1769 -r<R>: repeat the loop iteration <R> times and take the best result.
1760 1770 Default: 3
1761 1771
1762 1772 -t: use time.time to measure the time, which is the default on Unix.
1763 1773 This function measures wall time.
1764 1774
1765 1775 -c: use time.clock to measure the time, which is the default on
1766 1776 Windows and measures wall time. On Unix, resource.getrusage is used
1767 1777 instead and returns the CPU user time.
1768 1778
1769 1779 -p<P>: use a precision of <P> digits to display the timing result.
1770 1780 Default: 3
1771 1781
1772 1782
1773 1783 Examples:
1774 1784
1775 1785 In [1]: %timeit pass
1776 1786 10000000 loops, best of 3: 53.3 ns per loop
1777 1787
1778 1788 In [2]: u = None
1779 1789
1780 1790 In [3]: %timeit u is None
1781 1791 10000000 loops, best of 3: 184 ns per loop
1782 1792
1783 1793 In [4]: %timeit -r 4 u == None
1784 1794 1000000 loops, best of 4: 242 ns per loop
1785 1795
1786 1796 In [5]: import time
1787 1797
1788 1798 In [6]: %timeit -n1 time.sleep(2)
1789 1799 1 loops, best of 3: 2 s per loop
1790 1800
1791 1801
1792 1802 The times reported by %timeit will be slightly higher than those
1793 1803 reported by the timeit.py script when variables are accessed. This is
1794 1804 due to the fact that %timeit executes the statement in the namespace
1795 1805 of the shell, compared with timeit.py, which uses a single setup
1796 1806 statement to import function or create variables. Generally, the bias
1797 1807 does not matter as long as results from timeit.py are not mixed with
1798 1808 those from %timeit."""
1799 1809
1800 1810 import timeit
1801 1811 import math
1802 1812
1803 1813 # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
1804 1814 # certain terminals. Until we figure out a robust way of
1805 1815 # auto-detecting if the terminal can deal with it, use plain 'us' for
1806 1816 # microseconds. I am really NOT happy about disabling the proper
1807 1817 # 'micro' prefix, but crashing is worse... If anyone knows what the
1808 1818 # right solution for this is, I'm all ears...
1809 1819 #
1810 1820 # Note: using
1811 1821 #
1812 1822 # s = u'\xb5'
1813 1823 # s.encode(sys.getdefaultencoding())
1814 1824 #
1815 1825 # is not sufficient, as I've seen terminals where that fails but
1816 1826 # print s
1817 1827 #
1818 1828 # succeeds
1819 1829 #
1820 1830 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1821 1831
1822 1832 #units = [u"s", u"ms",u'\xb5',"ns"]
1823 1833 units = [u"s", u"ms",u'us',"ns"]
1824 1834
1825 1835 scaling = [1, 1e3, 1e6, 1e9]
1826 1836
1827 1837 opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
1828 1838 posix=False)
1829 1839 if stmt == "":
1830 1840 return
1831 1841 timefunc = timeit.default_timer
1832 1842 number = int(getattr(opts, "n", 0))
1833 1843 repeat = int(getattr(opts, "r", timeit.default_repeat))
1834 1844 precision = int(getattr(opts, "p", 3))
1835 1845 if hasattr(opts, "t"):
1836 1846 timefunc = time.time
1837 1847 if hasattr(opts, "c"):
1838 1848 timefunc = clock
1839 1849
1840 1850 timer = timeit.Timer(timer=timefunc)
1841 1851 # this code has tight coupling to the inner workings of timeit.Timer,
1842 1852 # but is there a better way to achieve that the code stmt has access
1843 1853 # to the shell namespace?
1844 1854
1845 1855 src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
1846 1856 'setup': "pass"}
1847 1857 # Track compilation time so it can be reported if too long
1848 1858 # Minimum time above which compilation time will be reported
1849 1859 tc_min = 0.1
1850 1860
1851 1861 t0 = clock()
1852 1862 code = compile(src, "<magic-timeit>", "exec")
1853 1863 tc = clock()-t0
1854 1864
1855 1865 ns = {}
1856 1866 exec code in self.shell.user_ns, ns
1857 1867 timer.inner = ns["inner"]
1858 1868
1859 1869 if number == 0:
1860 1870 # determine number so that 0.2 <= total time < 2.0
1861 1871 number = 1
1862 1872 for i in range(1, 10):
1863 1873 if timer.timeit(number) >= 0.2:
1864 1874 break
1865 1875 number *= 10
1866 1876
1867 1877 best = min(timer.repeat(repeat, number)) / number
1868 1878
1869 1879 if best > 0.0:
1870 1880 order = min(-int(math.floor(math.log10(best)) // 3), 3)
1871 1881 else:
1872 1882 order = 3
1873 1883 print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
1874 1884 precision,
1875 1885 best * scaling[order],
1876 1886 units[order])
1877 1887 if tc > tc_min:
1878 1888 print "Compiler time: %.2f s" % tc
1879 1889
1880 1890 @testdec.skip_doctest
1881 1891 def magic_time(self,parameter_s = ''):
1882 1892 """Time execution of a Python statement or expression.
1883 1893
1884 1894 The CPU and wall clock times are printed, and the value of the
1885 1895 expression (if any) is returned. Note that under Win32, system time
1886 1896 is always reported as 0, since it can not be measured.
1887 1897
1888 1898 This function provides very basic timing functionality. In Python
1889 1899 2.3, the timeit module offers more control and sophistication, so this
1890 1900 could be rewritten to use it (patches welcome).
1891 1901
1892 1902 Some examples:
1893 1903
1894 1904 In [1]: time 2**128
1895 1905 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1896 1906 Wall time: 0.00
1897 1907 Out[1]: 340282366920938463463374607431768211456L
1898 1908
1899 1909 In [2]: n = 1000000
1900 1910
1901 1911 In [3]: time sum(range(n))
1902 1912 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1903 1913 Wall time: 1.37
1904 1914 Out[3]: 499999500000L
1905 1915
1906 1916 In [4]: time print 'hello world'
1907 1917 hello world
1908 1918 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1909 1919 Wall time: 0.00
1910 1920
1911 1921 Note that the time needed by Python to compile the given expression
1912 1922 will be reported if it is more than 0.1s. In this example, the
1913 1923 actual exponentiation is done by Python at compilation time, so while
1914 1924 the expression can take a noticeable amount of time to compute, that
1915 1925 time is purely due to the compilation:
1916 1926
1917 1927 In [5]: time 3**9999;
1918 1928 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1919 1929 Wall time: 0.00 s
1920 1930
1921 1931 In [6]: time 3**999999;
1922 1932 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1923 1933 Wall time: 0.00 s
1924 1934 Compiler : 0.78 s
1925 1935 """
1926 1936
1927 1937 # fail immediately if the given expression can't be compiled
1928 1938
1929 1939 expr = self.shell.prefilter(parameter_s,False)
1930 1940
1931 1941 # Minimum time above which compilation time will be reported
1932 1942 tc_min = 0.1
1933 1943
1934 1944 try:
1935 1945 mode = 'eval'
1936 1946 t0 = clock()
1937 1947 code = compile(expr,'<timed eval>',mode)
1938 1948 tc = clock()-t0
1939 1949 except SyntaxError:
1940 1950 mode = 'exec'
1941 1951 t0 = clock()
1942 1952 code = compile(expr,'<timed exec>',mode)
1943 1953 tc = clock()-t0
1944 1954 # skew measurement as little as possible
1945 1955 glob = self.shell.user_ns
1946 1956 clk = clock2
1947 1957 wtime = time.time
1948 1958 # time execution
1949 1959 wall_st = wtime()
1950 1960 if mode=='eval':
1951 1961 st = clk()
1952 1962 out = eval(code,glob)
1953 1963 end = clk()
1954 1964 else:
1955 1965 st = clk()
1956 1966 exec code in glob
1957 1967 end = clk()
1958 1968 out = None
1959 1969 wall_end = wtime()
1960 1970 # Compute actual times and report
1961 1971 wall_time = wall_end-wall_st
1962 1972 cpu_user = end[0]-st[0]
1963 1973 cpu_sys = end[1]-st[1]
1964 1974 cpu_tot = cpu_user+cpu_sys
1965 1975 print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
1966 1976 (cpu_user,cpu_sys,cpu_tot)
1967 1977 print "Wall time: %.2f s" % wall_time
1968 1978 if tc > tc_min:
1969 1979 print "Compiler : %.2f s" % tc
1970 1980 return out
1971 1981
1972 1982 @testdec.skip_doctest
1973 1983 def magic_macro(self,parameter_s = ''):
1974 1984 """Define a set of input lines as a macro for future re-execution.
1975 1985
1976 1986 Usage:\\
1977 1987 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1978 1988
1979 1989 Options:
1980 1990
1981 1991 -r: use 'raw' input. By default, the 'processed' history is used,
1982 1992 so that magics are loaded in their transformed version to valid
1983 1993 Python. If this option is given, the raw input as typed as the
1984 1994 command line is used instead.
1985 1995
1986 1996 This will define a global variable called `name` which is a string
1987 1997 made of joining the slices and lines you specify (n1,n2,... numbers
1988 1998 above) from your input history into a single string. This variable
1989 1999 acts like an automatic function which re-executes those lines as if
1990 2000 you had typed them. You just type 'name' at the prompt and the code
1991 2001 executes.
1992 2002
1993 2003 The notation for indicating number ranges is: n1-n2 means 'use line
1994 2004 numbers n1,...n2' (the endpoint is included). That is, '5-7' means
1995 2005 using the lines numbered 5,6 and 7.
1996 2006
1997 2007 Note: as a 'hidden' feature, you can also use traditional python slice
1998 2008 notation, where N:M means numbers N through M-1.
1999 2009
2000 2010 For example, if your history contains (%hist prints it):
2001 2011
2002 2012 44: x=1
2003 2013 45: y=3
2004 2014 46: z=x+y
2005 2015 47: print x
2006 2016 48: a=5
2007 2017 49: print 'x',x,'y',y
2008 2018
2009 2019 you can create a macro with lines 44 through 47 (included) and line 49
2010 2020 called my_macro with:
2011 2021
2012 2022 In [55]: %macro my_macro 44-47 49
2013 2023
2014 2024 Now, typing `my_macro` (without quotes) will re-execute all this code
2015 2025 in one pass.
2016 2026
2017 2027 You don't need to give the line-numbers in order, and any given line
2018 2028 number can appear multiple times. You can assemble macros with any
2019 2029 lines from your input history in any order.
2020 2030
2021 2031 The macro is a simple object which holds its value in an attribute,
2022 2032 but IPython's display system checks for macros and executes them as
2023 2033 code instead of printing them when you type their name.
2024 2034
2025 2035 You can view a macro's contents by explicitly printing it with:
2026 2036
2027 2037 'print macro_name'.
2028 2038
2029 2039 For one-off cases which DON'T contain magic function calls in them you
2030 2040 can obtain similar results by explicitly executing slices from your
2031 2041 input history with:
2032 2042
2033 2043 In [60]: exec In[44:48]+In[49]"""
2034 2044
2035 2045 opts,args = self.parse_options(parameter_s,'r',mode='list')
2036 2046 if not args:
2037 2047 macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
2038 2048 macs.sort()
2039 2049 return macs
2040 2050 if len(args) == 1:
2041 2051 raise UsageError(
2042 2052 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
2043 2053 name,ranges = args[0], args[1:]
2044 2054
2045 2055 #print 'rng',ranges # dbg
2046 2056 lines = self.extract_input_slices(ranges,opts.has_key('r'))
2047 2057 macro = Macro(lines)
2048 2058 self.shell.user_ns.update({name:macro})
2049 2059 print 'Macro `%s` created. To execute, type its name (without quotes).' % name
2050 2060 print 'Macro contents:'
2051 2061 print macro,
2052 2062
2053 2063 def magic_save(self,parameter_s = ''):
2054 2064 """Save a set of lines to a given filename.
2055 2065
2056 2066 Usage:\\
2057 2067 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
2058 2068
2059 2069 Options:
2060 2070
2061 2071 -r: use 'raw' input. By default, the 'processed' history is used,
2062 2072 so that magics are loaded in their transformed version to valid
2063 2073 Python. If this option is given, the raw input as typed as the
2064 2074 command line is used instead.
2065 2075
2066 2076 This function uses the same syntax as %macro for line extraction, but
2067 2077 instead of creating a macro it saves the resulting string to the
2068 2078 filename you specify.
2069 2079
2070 2080 It adds a '.py' extension to the file if you don't do so yourself, and
2071 2081 it asks for confirmation before overwriting existing files."""
2072 2082
2073 2083 opts,args = self.parse_options(parameter_s,'r',mode='list')
2074 2084 fname,ranges = args[0], args[1:]
2075 2085 if not fname.endswith('.py'):
2076 2086 fname += '.py'
2077 2087 if os.path.isfile(fname):
2078 2088 ans = raw_input('File `%s` exists. Overwrite (y/[N])? ' % fname)
2079 2089 if ans.lower() not in ['y','yes']:
2080 2090 print 'Operation cancelled.'
2081 2091 return
2082 2092 cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
2083 2093 f = file(fname,'w')
2084 2094 f.write(cmds)
2085 2095 f.close()
2086 2096 print 'The following commands were written to file `%s`:' % fname
2087 2097 print cmds
2088 2098
2089 2099 def _edit_macro(self,mname,macro):
2090 2100 """open an editor with the macro data in a file"""
2091 2101 filename = self.shell.mktempfile(macro.value)
2092 2102 self.shell.hooks.editor(filename)
2093 2103
2094 2104 # and make a new macro object, to replace the old one
2095 2105 mfile = open(filename)
2096 2106 mvalue = mfile.read()
2097 2107 mfile.close()
2098 2108 self.shell.user_ns[mname] = Macro(mvalue)
2099 2109
2100 2110 def magic_ed(self,parameter_s=''):
2101 2111 """Alias to %edit."""
2102 2112 return self.magic_edit(parameter_s)
2103 2113
2104 2114 @testdec.skip_doctest
2105 2115 def magic_edit(self,parameter_s='',last_call=['','']):
2106 2116 """Bring up an editor and execute the resulting code.
2107 2117
2108 2118 Usage:
2109 2119 %edit [options] [args]
2110 2120
2111 2121 %edit runs IPython's editor hook. The default version of this hook is
2112 2122 set to call the __IPYTHON__.rc.editor command. This is read from your
2113 2123 environment variable $EDITOR. If this isn't found, it will default to
2114 2124 vi under Linux/Unix and to notepad under Windows. See the end of this
2115 2125 docstring for how to change the editor hook.
2116 2126
2117 2127 You can also set the value of this editor via the command line option
2118 2128 '-editor' or in your ipythonrc file. This is useful if you wish to use
2119 2129 specifically for IPython an editor different from your typical default
2120 2130 (and for Windows users who typically don't set environment variables).
2121 2131
2122 2132 This command allows you to conveniently edit multi-line code right in
2123 2133 your IPython session.
2124 2134
2125 2135 If called without arguments, %edit opens up an empty editor with a
2126 2136 temporary file and will execute the contents of this file when you
2127 2137 close it (don't forget to save it!).
2128 2138
2129 2139
2130 2140 Options:
2131 2141
2132 2142 -n <number>: open the editor at a specified line number. By default,
2133 2143 the IPython editor hook uses the unix syntax 'editor +N filename', but
2134 2144 you can configure this by providing your own modified hook if your
2135 2145 favorite editor supports line-number specifications with a different
2136 2146 syntax.
2137 2147
2138 2148 -p: this will call the editor with the same data as the previous time
2139 2149 it was used, regardless of how long ago (in your current session) it
2140 2150 was.
2141 2151
2142 2152 -r: use 'raw' input. This option only applies to input taken from the
2143 2153 user's history. By default, the 'processed' history is used, so that
2144 2154 magics are loaded in their transformed version to valid Python. If
2145 2155 this option is given, the raw input as typed as the command line is
2146 2156 used instead. When you exit the editor, it will be executed by
2147 2157 IPython's own processor.
2148 2158
2149 2159 -x: do not execute the edited code immediately upon exit. This is
2150 2160 mainly useful if you are editing programs which need to be called with
2151 2161 command line arguments, which you can then do using %run.
2152 2162
2153 2163
2154 2164 Arguments:
2155 2165
2156 2166 If arguments are given, the following possibilites exist:
2157 2167
2158 2168 - The arguments are numbers or pairs of colon-separated numbers (like
2159 2169 1 4:8 9). These are interpreted as lines of previous input to be
2160 2170 loaded into the editor. The syntax is the same of the %macro command.
2161 2171
2162 2172 - If the argument doesn't start with a number, it is evaluated as a
2163 2173 variable and its contents loaded into the editor. You can thus edit
2164 2174 any string which contains python code (including the result of
2165 2175 previous edits).
2166 2176
2167 2177 - If the argument is the name of an object (other than a string),
2168 2178 IPython will try to locate the file where it was defined and open the
2169 2179 editor at the point where it is defined. You can use `%edit function`
2170 2180 to load an editor exactly at the point where 'function' is defined,
2171 2181 edit it and have the file be executed automatically.
2172 2182
2173 2183 If the object is a macro (see %macro for details), this opens up your
2174 2184 specified editor with a temporary file containing the macro's data.
2175 2185 Upon exit, the macro is reloaded with the contents of the file.
2176 2186
2177 2187 Note: opening at an exact line is only supported under Unix, and some
2178 2188 editors (like kedit and gedit up to Gnome 2.8) do not understand the
2179 2189 '+NUMBER' parameter necessary for this feature. Good editors like
2180 2190 (X)Emacs, vi, jed, pico and joe all do.
2181 2191
2182 2192 - If the argument is not found as a variable, IPython will look for a
2183 2193 file with that name (adding .py if necessary) and load it into the
2184 2194 editor. It will execute its contents with execfile() when you exit,
2185 2195 loading any code in the file into your interactive namespace.
2186 2196
2187 2197 After executing your code, %edit will return as output the code you
2188 2198 typed in the editor (except when it was an existing file). This way
2189 2199 you can reload the code in further invocations of %edit as a variable,
2190 2200 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
2191 2201 the output.
2192 2202
2193 2203 Note that %edit is also available through the alias %ed.
2194 2204
2195 2205 This is an example of creating a simple function inside the editor and
2196 2206 then modifying it. First, start up the editor:
2197 2207
2198 2208 In [1]: ed
2199 2209 Editing... done. Executing edited code...
2200 2210 Out[1]: 'def foo():n print "foo() was defined in an editing session"n'
2201 2211
2202 2212 We can then call the function foo():
2203 2213
2204 2214 In [2]: foo()
2205 2215 foo() was defined in an editing session
2206 2216
2207 2217 Now we edit foo. IPython automatically loads the editor with the
2208 2218 (temporary) file where foo() was previously defined:
2209 2219
2210 2220 In [3]: ed foo
2211 2221 Editing... done. Executing edited code...
2212 2222
2213 2223 And if we call foo() again we get the modified version:
2214 2224
2215 2225 In [4]: foo()
2216 2226 foo() has now been changed!
2217 2227
2218 2228 Here is an example of how to edit a code snippet successive
2219 2229 times. First we call the editor:
2220 2230
2221 2231 In [5]: ed
2222 2232 Editing... done. Executing edited code...
2223 2233 hello
2224 2234 Out[5]: "print 'hello'n"
2225 2235
2226 2236 Now we call it again with the previous output (stored in _):
2227 2237
2228 2238 In [6]: ed _
2229 2239 Editing... done. Executing edited code...
2230 2240 hello world
2231 2241 Out[6]: "print 'hello world'n"
2232 2242
2233 2243 Now we call it with the output #8 (stored in _8, also as Out[8]):
2234 2244
2235 2245 In [7]: ed _8
2236 2246 Editing... done. Executing edited code...
2237 2247 hello again
2238 2248 Out[7]: "print 'hello again'n"
2239 2249
2240 2250
2241 2251 Changing the default editor hook:
2242 2252
2243 2253 If you wish to write your own editor hook, you can put it in a
2244 2254 configuration file which you load at startup time. The default hook
2245 2255 is defined in the IPython.hooks module, and you can use that as a
2246 2256 starting example for further modifications. That file also has
2247 2257 general instructions on how to set a new hook for use once you've
2248 2258 defined it."""
2249 2259
2250 2260 # FIXME: This function has become a convoluted mess. It needs a
2251 2261 # ground-up rewrite with clean, simple logic.
2252 2262
2253 2263 def make_filename(arg):
2254 2264 "Make a filename from the given args"
2255 2265 try:
2256 2266 filename = get_py_filename(arg)
2257 2267 except IOError:
2258 2268 if args.endswith('.py'):
2259 2269 filename = arg
2260 2270 else:
2261 2271 filename = None
2262 2272 return filename
2263 2273
2264 2274 # custom exceptions
2265 2275 class DataIsObject(Exception): pass
2266 2276
2267 2277 opts,args = self.parse_options(parameter_s,'prxn:')
2268 2278 # Set a few locals from the options for convenience:
2269 2279 opts_p = opts.has_key('p')
2270 2280 opts_r = opts.has_key('r')
2271 2281
2272 2282 # Default line number value
2273 2283 lineno = opts.get('n',None)
2274 2284
2275 2285 if opts_p:
2276 2286 args = '_%s' % last_call[0]
2277 2287 if not self.shell.user_ns.has_key(args):
2278 2288 args = last_call[1]
2279 2289
2280 2290 # use last_call to remember the state of the previous call, but don't
2281 2291 # let it be clobbered by successive '-p' calls.
2282 2292 try:
2283 2293 last_call[0] = self.shell.outputcache.prompt_count
2284 2294 if not opts_p:
2285 2295 last_call[1] = parameter_s
2286 2296 except:
2287 2297 pass
2288 2298
2289 2299 # by default this is done with temp files, except when the given
2290 2300 # arg is a filename
2291 2301 use_temp = 1
2292 2302
2293 2303 if re.match(r'\d',args):
2294 2304 # Mode where user specifies ranges of lines, like in %macro.
2295 2305 # This means that you can't edit files whose names begin with
2296 2306 # numbers this way. Tough.
2297 2307 ranges = args.split()
2298 2308 data = ''.join(self.extract_input_slices(ranges,opts_r))
2299 2309 elif args.endswith('.py'):
2300 2310 filename = make_filename(args)
2301 2311 data = ''
2302 2312 use_temp = 0
2303 2313 elif args:
2304 2314 try:
2305 2315 # Load the parameter given as a variable. If not a string,
2306 2316 # process it as an object instead (below)
2307 2317
2308 2318 #print '*** args',args,'type',type(args) # dbg
2309 2319 data = eval(args,self.shell.user_ns)
2310 2320 if not type(data) in StringTypes:
2311 2321 raise DataIsObject
2312 2322
2313 2323 except (NameError,SyntaxError):
2314 2324 # given argument is not a variable, try as a filename
2315 2325 filename = make_filename(args)
2316 2326 if filename is None:
2317 2327 warn("Argument given (%s) can't be found as a variable "
2318 2328 "or as a filename." % args)
2319 2329 return
2320 2330
2321 2331 data = ''
2322 2332 use_temp = 0
2323 2333 except DataIsObject:
2324 2334
2325 2335 # macros have a special edit function
2326 2336 if isinstance(data,Macro):
2327 2337 self._edit_macro(args,data)
2328 2338 return
2329 2339
2330 2340 # For objects, try to edit the file where they are defined
2331 2341 try:
2332 2342 filename = inspect.getabsfile(data)
2333 2343 if 'fakemodule' in filename.lower() and inspect.isclass(data):
2334 2344 # class created by %edit? Try to find source
2335 2345 # by looking for method definitions instead, the
2336 2346 # __module__ in those classes is FakeModule.
2337 2347 attrs = [getattr(data, aname) for aname in dir(data)]
2338 2348 for attr in attrs:
2339 2349 if not inspect.ismethod(attr):
2340 2350 continue
2341 2351 filename = inspect.getabsfile(attr)
2342 2352 if filename and 'fakemodule' not in filename.lower():
2343 2353 # change the attribute to be the edit target instead
2344 2354 data = attr
2345 2355 break
2346 2356
2347 2357 datafile = 1
2348 2358 except TypeError:
2349 2359 filename = make_filename(args)
2350 2360 datafile = 1
2351 2361 warn('Could not find file where `%s` is defined.\n'
2352 2362 'Opening a file named `%s`' % (args,filename))
2353 2363 # Now, make sure we can actually read the source (if it was in
2354 2364 # a temp file it's gone by now).
2355 2365 if datafile:
2356 2366 try:
2357 2367 if lineno is None:
2358 2368 lineno = inspect.getsourcelines(data)[1]
2359 2369 except IOError:
2360 2370 filename = make_filename(args)
2361 2371 if filename is None:
2362 2372 warn('The file `%s` where `%s` was defined cannot '
2363 2373 'be read.' % (filename,data))
2364 2374 return
2365 2375 use_temp = 0
2366 2376 else:
2367 2377 data = ''
2368 2378
2369 2379 if use_temp:
2370 2380 filename = self.shell.mktempfile(data)
2371 2381 print 'IPython will make a temporary file named:',filename
2372 2382
2373 2383 # do actual editing here
2374 2384 print 'Editing...',
2375 2385 sys.stdout.flush()
2376 2386 try:
2377 2387 self.shell.hooks.editor(filename,lineno)
2378 2388 except IPython.ipapi.TryNext:
2379 2389 warn('Could not open editor')
2380 2390 return
2381 2391
2382 2392 # XXX TODO: should this be generalized for all string vars?
2383 2393 # For now, this is special-cased to blocks created by cpaste
2384 2394 if args.strip() == 'pasted_block':
2385 2395 self.shell.user_ns['pasted_block'] = file_read(filename)
2386 2396
2387 2397 if opts.has_key('x'): # -x prevents actual execution
2388 2398 print
2389 2399 else:
2390 2400 print 'done. Executing edited code...'
2391 2401 if opts_r:
2392 2402 self.shell.runlines(file_read(filename))
2393 2403 else:
2394 2404 self.shell.safe_execfile(filename,self.shell.user_ns,
2395 2405 self.shell.user_ns)
2396 2406
2397 2407
2398 2408 if use_temp:
2399 2409 try:
2400 2410 return open(filename).read()
2401 2411 except IOError,msg:
2402 2412 if msg.filename == filename:
2403 2413 warn('File not found. Did you forget to save?')
2404 2414 return
2405 2415 else:
2406 2416 self.shell.showtraceback()
2407 2417
2408 2418 def magic_xmode(self,parameter_s = ''):
2409 2419 """Switch modes for the exception handlers.
2410 2420
2411 2421 Valid modes: Plain, Context and Verbose.
2412 2422
2413 2423 If called without arguments, acts as a toggle."""
2414 2424
2415 2425 def xmode_switch_err(name):
2416 2426 warn('Error changing %s exception modes.\n%s' %
2417 2427 (name,sys.exc_info()[1]))
2418 2428
2419 2429 shell = self.shell
2420 2430 new_mode = parameter_s.strip().capitalize()
2421 2431 try:
2422 2432 shell.InteractiveTB.set_mode(mode=new_mode)
2423 2433 print 'Exception reporting mode:',shell.InteractiveTB.mode
2424 2434 except:
2425 2435 xmode_switch_err('user')
2426 2436
2427 2437 # threaded shells use a special handler in sys.excepthook
2428 2438 if shell.isthreaded:
2429 2439 try:
2430 2440 shell.sys_excepthook.set_mode(mode=new_mode)
2431 2441 except:
2432 2442 xmode_switch_err('threaded')
2433 2443
2434 2444 def magic_colors(self,parameter_s = ''):
2435 2445 """Switch color scheme for prompts, info system and exception handlers.
2436 2446
2437 2447 Currently implemented schemes: NoColor, Linux, LightBG.
2438 2448
2439 2449 Color scheme names are not case-sensitive."""
2440 2450
2441 2451 def color_switch_err(name):
2442 2452 warn('Error changing %s color schemes.\n%s' %
2443 2453 (name,sys.exc_info()[1]))
2444 2454
2445 2455
2446 2456 new_scheme = parameter_s.strip()
2447 2457 if not new_scheme:
2448 2458 raise UsageError(
2449 2459 "%colors: you must specify a color scheme. See '%colors?'")
2450 2460 return
2451 2461 # local shortcut
2452 2462 shell = self.shell
2453 2463
2454 2464 import IPython.rlineimpl as readline
2455 2465
2456 2466 if not readline.have_readline and sys.platform == "win32":
2457 2467 msg = """\
2458 2468 Proper color support under MS Windows requires the pyreadline library.
2459 2469 You can find it at:
2460 2470 http://ipython.scipy.org/moin/PyReadline/Intro
2461 2471 Gary's readline needs the ctypes module, from:
2462 2472 http://starship.python.net/crew/theller/ctypes
2463 2473 (Note that ctypes is already part of Python versions 2.5 and newer).
2464 2474
2465 2475 Defaulting color scheme to 'NoColor'"""
2466 2476 new_scheme = 'NoColor'
2467 2477 warn(msg)
2468 2478
2469 2479 # readline option is 0
2470 2480 if not shell.has_readline:
2471 2481 new_scheme = 'NoColor'
2472 2482
2473 2483 # Set prompt colors
2474 2484 try:
2475 2485 shell.outputcache.set_colors(new_scheme)
2476 2486 except:
2477 2487 color_switch_err('prompt')
2478 2488 else:
2479 2489 shell.rc.colors = \
2480 2490 shell.outputcache.color_table.active_scheme_name
2481 2491 # Set exception colors
2482 2492 try:
2483 2493 shell.InteractiveTB.set_colors(scheme = new_scheme)
2484 2494 shell.SyntaxTB.set_colors(scheme = new_scheme)
2485 2495 except:
2486 2496 color_switch_err('exception')
2487 2497
2488 2498 # threaded shells use a verbose traceback in sys.excepthook
2489 2499 if shell.isthreaded:
2490 2500 try:
2491 2501 shell.sys_excepthook.set_colors(scheme=new_scheme)
2492 2502 except:
2493 2503 color_switch_err('system exception handler')
2494 2504
2495 2505 # Set info (for 'object?') colors
2496 2506 if shell.rc.color_info:
2497 2507 try:
2498 2508 shell.inspector.set_active_scheme(new_scheme)
2499 2509 except:
2500 2510 color_switch_err('object inspector')
2501 2511 else:
2502 2512 shell.inspector.set_active_scheme('NoColor')
2503 2513
2504 2514 def magic_color_info(self,parameter_s = ''):
2505 2515 """Toggle color_info.
2506 2516
2507 2517 The color_info configuration parameter controls whether colors are
2508 2518 used for displaying object details (by things like %psource, %pfile or
2509 2519 the '?' system). This function toggles this value with each call.
2510 2520
2511 2521 Note that unless you have a fairly recent pager (less works better
2512 2522 than more) in your system, using colored object information displays
2513 2523 will not work properly. Test it and see."""
2514 2524
2515 2525 self.shell.rc.color_info = 1 - self.shell.rc.color_info
2516 2526 self.magic_colors(self.shell.rc.colors)
2517 2527 print 'Object introspection functions have now coloring:',
2518 2528 print ['OFF','ON'][self.shell.rc.color_info]
2519 2529
2520 2530 def magic_Pprint(self, parameter_s=''):
2521 2531 """Toggle pretty printing on/off."""
2522 2532
2523 2533 self.shell.rc.pprint = 1 - self.shell.rc.pprint
2524 2534 print 'Pretty printing has been turned', \
2525 2535 ['OFF','ON'][self.shell.rc.pprint]
2526 2536
2527 2537 def magic_exit(self, parameter_s=''):
2528 2538 """Exit IPython, confirming if configured to do so.
2529 2539
2530 2540 You can configure whether IPython asks for confirmation upon exit by
2531 2541 setting the confirm_exit flag in the ipythonrc file."""
2532 2542
2533 2543 self.shell.exit()
2534 2544
2535 2545 def magic_quit(self, parameter_s=''):
2536 2546 """Exit IPython, confirming if configured to do so (like %exit)"""
2537 2547
2538 2548 self.shell.exit()
2539 2549
2540 2550 def magic_Exit(self, parameter_s=''):
2541 2551 """Exit IPython without confirmation."""
2542 2552
2543 2553 self.shell.ask_exit()
2544 2554
2545 2555 #......................................................................
2546 2556 # Functions to implement unix shell-type things
2547 2557
2548 2558 @testdec.skip_doctest
2549 2559 def magic_alias(self, parameter_s = ''):
2550 2560 """Define an alias for a system command.
2551 2561
2552 2562 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
2553 2563
2554 2564 Then, typing 'alias_name params' will execute the system command 'cmd
2555 2565 params' (from your underlying operating system).
2556 2566
2557 2567 Aliases have lower precedence than magic functions and Python normal
2558 2568 variables, so if 'foo' is both a Python variable and an alias, the
2559 2569 alias can not be executed until 'del foo' removes the Python variable.
2560 2570
2561 2571 You can use the %l specifier in an alias definition to represent the
2562 2572 whole line when the alias is called. For example:
2563 2573
2564 2574 In [2]: alias all echo "Input in brackets: <%l>"
2565 2575 In [3]: all hello world
2566 2576 Input in brackets: <hello world>
2567 2577
2568 2578 You can also define aliases with parameters using %s specifiers (one
2569 2579 per parameter):
2570 2580
2571 2581 In [1]: alias parts echo first %s second %s
2572 2582 In [2]: %parts A B
2573 2583 first A second B
2574 2584 In [3]: %parts A
2575 2585 Incorrect number of arguments: 2 expected.
2576 2586 parts is an alias to: 'echo first %s second %s'
2577 2587
2578 2588 Note that %l and %s are mutually exclusive. You can only use one or
2579 2589 the other in your aliases.
2580 2590
2581 2591 Aliases expand Python variables just like system calls using ! or !!
2582 2592 do: all expressions prefixed with '$' get expanded. For details of
2583 2593 the semantic rules, see PEP-215:
2584 2594 http://www.python.org/peps/pep-0215.html. This is the library used by
2585 2595 IPython for variable expansion. If you want to access a true shell
2586 2596 variable, an extra $ is necessary to prevent its expansion by IPython:
2587 2597
2588 2598 In [6]: alias show echo
2589 2599 In [7]: PATH='A Python string'
2590 2600 In [8]: show $PATH
2591 2601 A Python string
2592 2602 In [9]: show $$PATH
2593 2603 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
2594 2604
2595 2605 You can use the alias facility to acess all of $PATH. See the %rehash
2596 2606 and %rehashx functions, which automatically create aliases for the
2597 2607 contents of your $PATH.
2598 2608
2599 2609 If called with no parameters, %alias prints the current alias table."""
2600 2610
2601 2611 par = parameter_s.strip()
2602 2612 if not par:
2603 2613 stored = self.db.get('stored_aliases', {} )
2604 2614 atab = self.shell.alias_table
2605 2615 aliases = atab.keys()
2606 2616 aliases.sort()
2607 2617 res = []
2608 2618 showlast = []
2609 2619 for alias in aliases:
2610 2620 special = False
2611 2621 try:
2612 2622 tgt = atab[alias][1]
2613 2623 except (TypeError, AttributeError):
2614 2624 # unsubscriptable? probably a callable
2615 2625 tgt = atab[alias]
2616 2626 special = True
2617 2627 # 'interesting' aliases
2618 2628 if (alias in stored or
2619 2629 special or
2620 2630 alias.lower() != os.path.splitext(tgt)[0].lower() or
2621 2631 ' ' in tgt):
2622 2632 showlast.append((alias, tgt))
2623 2633 else:
2624 2634 res.append((alias, tgt ))
2625 2635
2626 2636 # show most interesting aliases last
2627 2637 res.extend(showlast)
2628 2638 print "Total number of aliases:",len(aliases)
2629 2639 return res
2630 2640 try:
2631 2641 alias,cmd = par.split(None,1)
2632 2642 except:
2633 2643 print OInspect.getdoc(self.magic_alias)
2634 2644 else:
2635 2645 nargs = cmd.count('%s')
2636 2646 if nargs>0 and cmd.find('%l')>=0:
2637 2647 error('The %s and %l specifiers are mutually exclusive '
2638 2648 'in alias definitions.')
2639 2649 else: # all looks OK
2640 2650 self.shell.alias_table[alias] = (nargs,cmd)
2641 2651 self.shell.alias_table_validate(verbose=0)
2642 2652 # end magic_alias
2643 2653
2644 2654 def magic_unalias(self, parameter_s = ''):
2645 2655 """Remove an alias"""
2646 2656
2647 2657 aname = parameter_s.strip()
2648 2658 if aname in self.shell.alias_table:
2649 2659 del self.shell.alias_table[aname]
2650 2660 stored = self.db.get('stored_aliases', {} )
2651 2661 if aname in stored:
2652 2662 print "Removing %stored alias",aname
2653 2663 del stored[aname]
2654 2664 self.db['stored_aliases'] = stored
2655 2665
2656 2666
2657 2667 def magic_rehashx(self, parameter_s = ''):
2658 2668 """Update the alias table with all executable files in $PATH.
2659 2669
2660 2670 This version explicitly checks that every entry in $PATH is a file
2661 2671 with execute access (os.X_OK), so it is much slower than %rehash.
2662 2672
2663 2673 Under Windows, it checks executability as a match agains a
2664 2674 '|'-separated string of extensions, stored in the IPython config
2665 2675 variable win_exec_ext. This defaults to 'exe|com|bat'.
2666 2676
2667 2677 This function also resets the root module cache of module completer,
2668 2678 used on slow filesystems.
2669 2679 """
2670 2680
2671 2681
2672 2682 ip = self.api
2673 2683
2674 2684 # for the benefit of module completer in ipy_completers.py
2675 2685 del ip.db['rootmodules']
2676 2686
2677 2687 path = [os.path.abspath(os.path.expanduser(p)) for p in
2678 2688 os.environ.get('PATH','').split(os.pathsep)]
2679 2689 path = filter(os.path.isdir,path)
2680 2690
2681 2691 alias_table = self.shell.alias_table
2682 2692 syscmdlist = []
2683 2693 if os.name == 'posix':
2684 2694 isexec = lambda fname:os.path.isfile(fname) and \
2685 2695 os.access(fname,os.X_OK)
2686 2696 else:
2687 2697
2688 2698 try:
2689 2699 winext = os.environ['pathext'].replace(';','|').replace('.','')
2690 2700 except KeyError:
2691 2701 winext = 'exe|com|bat|py'
2692 2702 if 'py' not in winext:
2693 2703 winext += '|py'
2694 2704 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
2695 2705 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
2696 2706 savedir = os.getcwd()
2697 2707 try:
2698 2708 # write the whole loop for posix/Windows so we don't have an if in
2699 2709 # the innermost part
2700 2710 if os.name == 'posix':
2701 2711 for pdir in path:
2702 2712 os.chdir(pdir)
2703 2713 for ff in os.listdir(pdir):
2704 2714 if isexec(ff) and ff not in self.shell.no_alias:
2705 2715 # each entry in the alias table must be (N,name),
2706 2716 # where N is the number of positional arguments of the
2707 2717 # alias.
2708 2718 # Dots will be removed from alias names, since ipython
2709 2719 # assumes names with dots to be python code
2710 2720 alias_table[ff.replace('.','')] = (0,ff)
2711 2721 syscmdlist.append(ff)
2712 2722 else:
2713 2723 for pdir in path:
2714 2724 os.chdir(pdir)
2715 2725 for ff in os.listdir(pdir):
2716 2726 base, ext = os.path.splitext(ff)
2717 2727 if isexec(ff) and base.lower() not in self.shell.no_alias:
2718 2728 if ext.lower() == '.exe':
2719 2729 ff = base
2720 2730 alias_table[base.lower().replace('.','')] = (0,ff)
2721 2731 syscmdlist.append(ff)
2722 2732 # Make sure the alias table doesn't contain keywords or builtins
2723 2733 self.shell.alias_table_validate()
2724 2734 # Call again init_auto_alias() so we get 'rm -i' and other
2725 2735 # modified aliases since %rehashx will probably clobber them
2726 2736
2727 2737 # no, we don't want them. if %rehashx clobbers them, good,
2728 2738 # we'll probably get better versions
2729 2739 # self.shell.init_auto_alias()
2730 2740 db = ip.db
2731 2741 db['syscmdlist'] = syscmdlist
2732 2742 finally:
2733 2743 os.chdir(savedir)
2734 2744
2735 2745 def magic_pwd(self, parameter_s = ''):
2736 2746 """Return the current working directory path."""
2737 2747 return os.getcwd()
2738 2748
2739 2749 def magic_cd(self, parameter_s=''):
2740 2750 """Change the current working directory.
2741 2751
2742 2752 This command automatically maintains an internal list of directories
2743 2753 you visit during your IPython session, in the variable _dh. The
2744 2754 command %dhist shows this history nicely formatted. You can also
2745 2755 do 'cd -<tab>' to see directory history conveniently.
2746 2756
2747 2757 Usage:
2748 2758
2749 2759 cd 'dir': changes to directory 'dir'.
2750 2760
2751 2761 cd -: changes to the last visited directory.
2752 2762
2753 2763 cd -<n>: changes to the n-th directory in the directory history.
2754 2764
2755 2765 cd --foo: change to directory that matches 'foo' in history
2756 2766
2757 2767 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
2758 2768 (note: cd <bookmark_name> is enough if there is no
2759 2769 directory <bookmark_name>, but a bookmark with the name exists.)
2760 2770 'cd -b <tab>' allows you to tab-complete bookmark names.
2761 2771
2762 2772 Options:
2763 2773
2764 2774 -q: quiet. Do not print the working directory after the cd command is
2765 2775 executed. By default IPython's cd command does print this directory,
2766 2776 since the default prompts do not display path information.
2767 2777
2768 2778 Note that !cd doesn't work for this purpose because the shell where
2769 2779 !command runs is immediately discarded after executing 'command'."""
2770 2780
2771 2781 parameter_s = parameter_s.strip()
2772 2782 #bkms = self.shell.persist.get("bookmarks",{})
2773 2783
2774 2784 oldcwd = os.getcwd()
2775 2785 numcd = re.match(r'(-)(\d+)$',parameter_s)
2776 2786 # jump in directory history by number
2777 2787 if numcd:
2778 2788 nn = int(numcd.group(2))
2779 2789 try:
2780 2790 ps = self.shell.user_ns['_dh'][nn]
2781 2791 except IndexError:
2782 2792 print 'The requested directory does not exist in history.'
2783 2793 return
2784 2794 else:
2785 2795 opts = {}
2786 2796 elif parameter_s.startswith('--'):
2787 2797 ps = None
2788 2798 fallback = None
2789 2799 pat = parameter_s[2:]
2790 2800 dh = self.shell.user_ns['_dh']
2791 2801 # first search only by basename (last component)
2792 2802 for ent in reversed(dh):
2793 2803 if pat in os.path.basename(ent) and os.path.isdir(ent):
2794 2804 ps = ent
2795 2805 break
2796 2806
2797 2807 if fallback is None and pat in ent and os.path.isdir(ent):
2798 2808 fallback = ent
2799 2809
2800 2810 # if we have no last part match, pick the first full path match
2801 2811 if ps is None:
2802 2812 ps = fallback
2803 2813
2804 2814 if ps is None:
2805 2815 print "No matching entry in directory history"
2806 2816 return
2807 2817 else:
2808 2818 opts = {}
2809 2819
2810 2820
2811 2821 else:
2812 2822 #turn all non-space-escaping backslashes to slashes,
2813 2823 # for c:\windows\directory\names\
2814 2824 parameter_s = re.sub(r'\\(?! )','/', parameter_s)
2815 2825 opts,ps = self.parse_options(parameter_s,'qb',mode='string')
2816 2826 # jump to previous
2817 2827 if ps == '-':
2818 2828 try:
2819 2829 ps = self.shell.user_ns['_dh'][-2]
2820 2830 except IndexError:
2821 2831 raise UsageError('%cd -: No previous directory to change to.')
2822 2832 # jump to bookmark if needed
2823 2833 else:
2824 2834 if not os.path.isdir(ps) or opts.has_key('b'):
2825 2835 bkms = self.db.get('bookmarks', {})
2826 2836
2827 2837 if bkms.has_key(ps):
2828 2838 target = bkms[ps]
2829 2839 print '(bookmark:%s) -> %s' % (ps,target)
2830 2840 ps = target
2831 2841 else:
2832 2842 if opts.has_key('b'):
2833 2843 raise UsageError("Bookmark '%s' not found. "
2834 2844 "Use '%%bookmark -l' to see your bookmarks." % ps)
2835 2845
2836 2846 # at this point ps should point to the target dir
2837 2847 if ps:
2838 2848 try:
2839 2849 os.chdir(os.path.expanduser(ps))
2840 2850 if self.shell.rc.term_title:
2841 2851 #print 'set term title:',self.shell.rc.term_title # dbg
2842 2852 platutils.set_term_title('IPy ' + abbrev_cwd())
2843 2853 except OSError:
2844 2854 print sys.exc_info()[1]
2845 2855 else:
2846 2856 cwd = os.getcwd()
2847 2857 dhist = self.shell.user_ns['_dh']
2848 2858 if oldcwd != cwd:
2849 2859 dhist.append(cwd)
2850 2860 self.db['dhist'] = compress_dhist(dhist)[-100:]
2851 2861
2852 2862 else:
2853 2863 os.chdir(self.shell.home_dir)
2854 2864 if self.shell.rc.term_title:
2855 2865 platutils.set_term_title("IPy ~")
2856 2866 cwd = os.getcwd()
2857 2867 dhist = self.shell.user_ns['_dh']
2858 2868
2859 2869 if oldcwd != cwd:
2860 2870 dhist.append(cwd)
2861 2871 self.db['dhist'] = compress_dhist(dhist)[-100:]
2862 2872 if not 'q' in opts and self.shell.user_ns['_dh']:
2863 2873 print self.shell.user_ns['_dh'][-1]
2864 2874
2865 2875
2866 2876 def magic_env(self, parameter_s=''):
2867 2877 """List environment variables."""
2868 2878
2869 2879 return os.environ.data
2870 2880
2871 2881 def magic_pushd(self, parameter_s=''):
2872 2882 """Place the current dir on stack and change directory.
2873 2883
2874 2884 Usage:\\
2875 2885 %pushd ['dirname']
2876 2886 """
2877 2887
2878 2888 dir_s = self.shell.dir_stack
2879 2889 tgt = os.path.expanduser(parameter_s)
2880 2890 cwd = os.getcwd().replace(self.home_dir,'~')
2881 2891 if tgt:
2882 2892 self.magic_cd(parameter_s)
2883 2893 dir_s.insert(0,cwd)
2884 2894 return self.magic_dirs()
2885 2895
2886 2896 def magic_popd(self, parameter_s=''):
2887 2897 """Change to directory popped off the top of the stack.
2888 2898 """
2889 2899 if not self.shell.dir_stack:
2890 2900 raise UsageError("%popd on empty stack")
2891 2901 top = self.shell.dir_stack.pop(0)
2892 2902 self.magic_cd(top)
2893 2903 print "popd ->",top
2894 2904
2895 2905 def magic_dirs(self, parameter_s=''):
2896 2906 """Return the current directory stack."""
2897 2907
2898 2908 return self.shell.dir_stack
2899 2909
2900 2910 def magic_dhist(self, parameter_s=''):
2901 2911 """Print your history of visited directories.
2902 2912
2903 2913 %dhist -> print full history\\
2904 2914 %dhist n -> print last n entries only\\
2905 2915 %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
2906 2916
2907 2917 This history is automatically maintained by the %cd command, and
2908 2918 always available as the global list variable _dh. You can use %cd -<n>
2909 2919 to go to directory number <n>.
2910 2920
2911 2921 Note that most of time, you should view directory history by entering
2912 2922 cd -<TAB>.
2913 2923
2914 2924 """
2915 2925
2916 2926 dh = self.shell.user_ns['_dh']
2917 2927 if parameter_s:
2918 2928 try:
2919 2929 args = map(int,parameter_s.split())
2920 2930 except:
2921 2931 self.arg_err(Magic.magic_dhist)
2922 2932 return
2923 2933 if len(args) == 1:
2924 2934 ini,fin = max(len(dh)-(args[0]),0),len(dh)
2925 2935 elif len(args) == 2:
2926 2936 ini,fin = args
2927 2937 else:
2928 2938 self.arg_err(Magic.magic_dhist)
2929 2939 return
2930 2940 else:
2931 2941 ini,fin = 0,len(dh)
2932 2942 nlprint(dh,
2933 2943 header = 'Directory history (kept in _dh)',
2934 2944 start=ini,stop=fin)
2935 2945
2936 2946 @testdec.skip_doctest
2937 2947 def magic_sc(self, parameter_s=''):
2938 2948 """Shell capture - execute a shell command and capture its output.
2939 2949
2940 2950 DEPRECATED. Suboptimal, retained for backwards compatibility.
2941 2951
2942 2952 You should use the form 'var = !command' instead. Example:
2943 2953
2944 2954 "%sc -l myfiles = ls ~" should now be written as
2945 2955
2946 2956 "myfiles = !ls ~"
2947 2957
2948 2958 myfiles.s, myfiles.l and myfiles.n still apply as documented
2949 2959 below.
2950 2960
2951 2961 --
2952 2962 %sc [options] varname=command
2953 2963
2954 2964 IPython will run the given command using commands.getoutput(), and
2955 2965 will then update the user's interactive namespace with a variable
2956 2966 called varname, containing the value of the call. Your command can
2957 2967 contain shell wildcards, pipes, etc.
2958 2968
2959 2969 The '=' sign in the syntax is mandatory, and the variable name you
2960 2970 supply must follow Python's standard conventions for valid names.
2961 2971
2962 2972 (A special format without variable name exists for internal use)
2963 2973
2964 2974 Options:
2965 2975
2966 2976 -l: list output. Split the output on newlines into a list before
2967 2977 assigning it to the given variable. By default the output is stored
2968 2978 as a single string.
2969 2979
2970 2980 -v: verbose. Print the contents of the variable.
2971 2981
2972 2982 In most cases you should not need to split as a list, because the
2973 2983 returned value is a special type of string which can automatically
2974 2984 provide its contents either as a list (split on newlines) or as a
2975 2985 space-separated string. These are convenient, respectively, either
2976 2986 for sequential processing or to be passed to a shell command.
2977 2987
2978 2988 For example:
2979 2989
2980 2990 # all-random
2981 2991
2982 2992 # Capture into variable a
2983 2993 In [1]: sc a=ls *py
2984 2994
2985 2995 # a is a string with embedded newlines
2986 2996 In [2]: a
2987 2997 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
2988 2998
2989 2999 # which can be seen as a list:
2990 3000 In [3]: a.l
2991 3001 Out[3]: ['setup.py', 'win32_manual_post_install.py']
2992 3002
2993 3003 # or as a whitespace-separated string:
2994 3004 In [4]: a.s
2995 3005 Out[4]: 'setup.py win32_manual_post_install.py'
2996 3006
2997 3007 # a.s is useful to pass as a single command line:
2998 3008 In [5]: !wc -l $a.s
2999 3009 146 setup.py
3000 3010 130 win32_manual_post_install.py
3001 3011 276 total
3002 3012
3003 3013 # while the list form is useful to loop over:
3004 3014 In [6]: for f in a.l:
3005 3015 ...: !wc -l $f
3006 3016 ...:
3007 3017 146 setup.py
3008 3018 130 win32_manual_post_install.py
3009 3019
3010 3020 Similiarly, the lists returned by the -l option are also special, in
3011 3021 the sense that you can equally invoke the .s attribute on them to
3012 3022 automatically get a whitespace-separated string from their contents:
3013 3023
3014 3024 In [7]: sc -l b=ls *py
3015 3025
3016 3026 In [8]: b
3017 3027 Out[8]: ['setup.py', 'win32_manual_post_install.py']
3018 3028
3019 3029 In [9]: b.s
3020 3030 Out[9]: 'setup.py win32_manual_post_install.py'
3021 3031
3022 3032 In summary, both the lists and strings used for ouptut capture have
3023 3033 the following special attributes:
3024 3034
3025 3035 .l (or .list) : value as list.
3026 3036 .n (or .nlstr): value as newline-separated string.
3027 3037 .s (or .spstr): value as space-separated string.
3028 3038 """
3029 3039
3030 3040 opts,args = self.parse_options(parameter_s,'lv')
3031 3041 # Try to get a variable name and command to run
3032 3042 try:
3033 3043 # the variable name must be obtained from the parse_options
3034 3044 # output, which uses shlex.split to strip options out.
3035 3045 var,_ = args.split('=',1)
3036 3046 var = var.strip()
3037 3047 # But the the command has to be extracted from the original input
3038 3048 # parameter_s, not on what parse_options returns, to avoid the
3039 3049 # quote stripping which shlex.split performs on it.
3040 3050 _,cmd = parameter_s.split('=',1)
3041 3051 except ValueError:
3042 3052 var,cmd = '',''
3043 3053 # If all looks ok, proceed
3044 3054 out,err = self.shell.getoutputerror(cmd)
3045 3055 if err:
3046 3056 print >> Term.cerr,err
3047 3057 if opts.has_key('l'):
3048 3058 out = SList(out.split('\n'))
3049 3059 else:
3050 3060 out = LSString(out)
3051 3061 if opts.has_key('v'):
3052 3062 print '%s ==\n%s' % (var,pformat(out))
3053 3063 if var:
3054 3064 self.shell.user_ns.update({var:out})
3055 3065 else:
3056 3066 return out
3057 3067
3058 3068 def magic_sx(self, parameter_s=''):
3059 3069 """Shell execute - run a shell command and capture its output.
3060 3070
3061 3071 %sx command
3062 3072
3063 3073 IPython will run the given command using commands.getoutput(), and
3064 3074 return the result formatted as a list (split on '\\n'). Since the
3065 3075 output is _returned_, it will be stored in ipython's regular output
3066 3076 cache Out[N] and in the '_N' automatic variables.
3067 3077
3068 3078 Notes:
3069 3079
3070 3080 1) If an input line begins with '!!', then %sx is automatically
3071 3081 invoked. That is, while:
3072 3082 !ls
3073 3083 causes ipython to simply issue system('ls'), typing
3074 3084 !!ls
3075 3085 is a shorthand equivalent to:
3076 3086 %sx ls
3077 3087
3078 3088 2) %sx differs from %sc in that %sx automatically splits into a list,
3079 3089 like '%sc -l'. The reason for this is to make it as easy as possible
3080 3090 to process line-oriented shell output via further python commands.
3081 3091 %sc is meant to provide much finer control, but requires more
3082 3092 typing.
3083 3093
3084 3094 3) Just like %sc -l, this is a list with special attributes:
3085 3095
3086 3096 .l (or .list) : value as list.
3087 3097 .n (or .nlstr): value as newline-separated string.
3088 3098 .s (or .spstr): value as whitespace-separated string.
3089 3099
3090 3100 This is very useful when trying to use such lists as arguments to
3091 3101 system commands."""
3092 3102
3093 3103 if parameter_s:
3094 3104 out,err = self.shell.getoutputerror(parameter_s)
3095 3105 if err:
3096 3106 print >> Term.cerr,err
3097 3107 return SList(out.split('\n'))
3098 3108
3099 3109 def magic_bg(self, parameter_s=''):
3100 3110 """Run a job in the background, in a separate thread.
3101 3111
3102 3112 For example,
3103 3113
3104 3114 %bg myfunc(x,y,z=1)
3105 3115
3106 3116 will execute 'myfunc(x,y,z=1)' in a background thread. As soon as the
3107 3117 execution starts, a message will be printed indicating the job
3108 3118 number. If your job number is 5, you can use
3109 3119
3110 3120 myvar = jobs.result(5) or myvar = jobs[5].result
3111 3121
3112 3122 to assign this result to variable 'myvar'.
3113 3123
3114 3124 IPython has a job manager, accessible via the 'jobs' object. You can
3115 3125 type jobs? to get more information about it, and use jobs.<TAB> to see
3116 3126 its attributes. All attributes not starting with an underscore are
3117 3127 meant for public use.
3118 3128
3119 3129 In particular, look at the jobs.new() method, which is used to create
3120 3130 new jobs. This magic %bg function is just a convenience wrapper
3121 3131 around jobs.new(), for expression-based jobs. If you want to create a
3122 3132 new job with an explicit function object and arguments, you must call
3123 3133 jobs.new() directly.
3124 3134
3125 3135 The jobs.new docstring also describes in detail several important
3126 3136 caveats associated with a thread-based model for background job
3127 3137 execution. Type jobs.new? for details.
3128 3138
3129 3139 You can check the status of all jobs with jobs.status().
3130 3140
3131 3141 The jobs variable is set by IPython into the Python builtin namespace.
3132 3142 If you ever declare a variable named 'jobs', you will shadow this
3133 3143 name. You can either delete your global jobs variable to regain
3134 3144 access to the job manager, or make a new name and assign it manually
3135 3145 to the manager (stored in IPython's namespace). For example, to
3136 3146 assign the job manager to the Jobs name, use:
3137 3147
3138 3148 Jobs = __builtins__.jobs"""
3139 3149
3140 3150 self.shell.jobs.new(parameter_s,self.shell.user_ns)
3141 3151
3142 3152 def magic_r(self, parameter_s=''):
3143 3153 """Repeat previous input.
3144 3154
3145 3155 Note: Consider using the more powerfull %rep instead!
3146 3156
3147 3157 If given an argument, repeats the previous command which starts with
3148 3158 the same string, otherwise it just repeats the previous input.
3149 3159
3150 3160 Shell escaped commands (with ! as first character) are not recognized
3151 3161 by this system, only pure python code and magic commands.
3152 3162 """
3153 3163
3154 3164 start = parameter_s.strip()
3155 3165 esc_magic = self.shell.ESC_MAGIC
3156 3166 # Identify magic commands even if automagic is on (which means
3157 3167 # the in-memory version is different from that typed by the user).
3158 3168 if self.shell.rc.automagic:
3159 3169 start_magic = esc_magic+start
3160 3170 else:
3161 3171 start_magic = start
3162 3172 # Look through the input history in reverse
3163 3173 for n in range(len(self.shell.input_hist)-2,0,-1):
3164 3174 input = self.shell.input_hist[n]
3165 3175 # skip plain 'r' lines so we don't recurse to infinity
3166 3176 if input != '_ip.magic("r")\n' and \
3167 3177 (input.startswith(start) or input.startswith(start_magic)):
3168 3178 #print 'match',`input` # dbg
3169 3179 print 'Executing:',input,
3170 3180 self.shell.runlines(input)
3171 3181 return
3172 3182 print 'No previous input matching `%s` found.' % start
3173 3183
3174 3184
3175 3185 def magic_bookmark(self, parameter_s=''):
3176 3186 """Manage IPython's bookmark system.
3177 3187
3178 3188 %bookmark <name> - set bookmark to current dir
3179 3189 %bookmark <name> <dir> - set bookmark to <dir>
3180 3190 %bookmark -l - list all bookmarks
3181 3191 %bookmark -d <name> - remove bookmark
3182 3192 %bookmark -r - remove all bookmarks
3183 3193
3184 3194 You can later on access a bookmarked folder with:
3185 3195 %cd -b <name>
3186 3196 or simply '%cd <name>' if there is no directory called <name> AND
3187 3197 there is such a bookmark defined.
3188 3198
3189 3199 Your bookmarks persist through IPython sessions, but they are
3190 3200 associated with each profile."""
3191 3201
3192 3202 opts,args = self.parse_options(parameter_s,'drl',mode='list')
3193 3203 if len(args) > 2:
3194 3204 raise UsageError("%bookmark: too many arguments")
3195 3205
3196 3206 bkms = self.db.get('bookmarks',{})
3197 3207
3198 3208 if opts.has_key('d'):
3199 3209 try:
3200 3210 todel = args[0]
3201 3211 except IndexError:
3202 3212 raise UsageError(
3203 3213 "%bookmark -d: must provide a bookmark to delete")
3204 3214 else:
3205 3215 try:
3206 3216 del bkms[todel]
3207 3217 except KeyError:
3208 3218 raise UsageError(
3209 3219 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
3210 3220
3211 3221 elif opts.has_key('r'):
3212 3222 bkms = {}
3213 3223 elif opts.has_key('l'):
3214 3224 bks = bkms.keys()
3215 3225 bks.sort()
3216 3226 if bks:
3217 3227 size = max(map(len,bks))
3218 3228 else:
3219 3229 size = 0
3220 3230 fmt = '%-'+str(size)+'s -> %s'
3221 3231 print 'Current bookmarks:'
3222 3232 for bk in bks:
3223 3233 print fmt % (bk,bkms[bk])
3224 3234 else:
3225 3235 if not args:
3226 3236 raise UsageError("%bookmark: You must specify the bookmark name")
3227 3237 elif len(args)==1:
3228 3238 bkms[args[0]] = os.getcwd()
3229 3239 elif len(args)==2:
3230 3240 bkms[args[0]] = args[1]
3231 3241 self.db['bookmarks'] = bkms
3232 3242
3233 3243 def magic_pycat(self, parameter_s=''):
3234 3244 """Show a syntax-highlighted file through a pager.
3235 3245
3236 3246 This magic is similar to the cat utility, but it will assume the file
3237 3247 to be Python source and will show it with syntax highlighting. """
3238 3248
3239 3249 try:
3240 3250 filename = get_py_filename(parameter_s)
3241 3251 cont = file_read(filename)
3242 3252 except IOError:
3243 3253 try:
3244 3254 cont = eval(parameter_s,self.user_ns)
3245 3255 except NameError:
3246 3256 cont = None
3247 3257 if cont is None:
3248 3258 print "Error: no such file or variable"
3249 3259 return
3250 3260
3251 3261 page(self.shell.pycolorize(cont),
3252 3262 screen_lines=self.shell.rc.screen_length)
3253 3263
3254 3264 def magic_cpaste(self, parameter_s=''):
3255 3265 """Allows you to paste & execute a pre-formatted code block from clipboard.
3256 3266
3257 3267 You must terminate the block with '--' (two minus-signs) alone on the
3258 3268 line. You can also provide your own sentinel with '%paste -s %%' ('%%'
3259 3269 is the new sentinel for this operation)
3260 3270
3261 3271 The block is dedented prior to execution to enable execution of method
3262 3272 definitions. '>' and '+' characters at the beginning of a line are
3263 3273 ignored, to allow pasting directly from e-mails, diff files and
3264 3274 doctests (the '...' continuation prompt is also stripped). The
3265 3275 executed block is also assigned to variable named 'pasted_block' for
3266 3276 later editing with '%edit pasted_block'.
3267 3277
3268 3278 You can also pass a variable name as an argument, e.g. '%cpaste foo'.
3269 3279 This assigns the pasted block to variable 'foo' as string, without
3270 3280 dedenting or executing it (preceding >>> and + is still stripped)
3271 3281
3272 3282 '%cpaste -r' re-executes the block previously entered by cpaste.
3273 3283
3274 3284 Do not be alarmed by garbled output on Windows (it's a readline bug).
3275 3285 Just press enter and type -- (and press enter again) and the block
3276 3286 will be what was just pasted.
3277 3287
3278 3288 IPython statements (magics, shell escapes) are not supported (yet).
3279 3289 """
3280 3290 opts,args = self.parse_options(parameter_s,'rs:',mode='string')
3281 3291 par = args.strip()
3282 3292 if opts.has_key('r'):
3283 3293 b = self.user_ns.get('pasted_block', None)
3284 3294 if b is None:
3285 3295 raise UsageError('No previous pasted block available')
3286 3296 print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
3287 3297 exec b in self.user_ns
3288 3298 return
3289 3299
3290 3300 sentinel = opts.get('s','--')
3291 3301
3292 3302 # Regular expressions that declare text we strip from the input:
3293 3303 strip_re = [r'^\s*In \[\d+\]:', # IPython input prompt
3294 3304 r'^\s*(\s?>)+', # Python input prompt
3295 3305 r'^\s*\.{3,}', # Continuation prompts
3296 3306 r'^\++',
3297 3307 ]
3298 3308
3299 3309 strip_from_start = map(re.compile,strip_re)
3300 3310
3301 3311 from IPython import iplib
3302 3312 lines = []
3303 3313 print "Pasting code; enter '%s' alone on the line to stop." % sentinel
3304 3314 while 1:
3305 3315 l = iplib.raw_input_original(':')
3306 3316 if l ==sentinel:
3307 3317 break
3308 3318
3309 3319 for pat in strip_from_start:
3310 3320 l = pat.sub('',l)
3311 3321 lines.append(l)
3312 3322
3313 3323 block = "\n".join(lines) + '\n'
3314 3324 #print "block:\n",block
3315 3325 if not par:
3316 3326 b = textwrap.dedent(block)
3317 3327 self.user_ns['pasted_block'] = b
3318 3328 exec b in self.user_ns
3319 3329 else:
3320 3330 self.user_ns[par] = SList(block.splitlines())
3321 3331 print "Block assigned to '%s'" % par
3322 3332
3323 3333 def magic_quickref(self,arg):
3324 3334 """ Show a quick reference sheet """
3325 3335 import IPython.usage
3326 3336 qr = IPython.usage.quick_reference + self.magic_magic('-brief')
3327 3337
3328 3338 page(qr)
3329 3339
3330 3340 def magic_upgrade(self,arg):
3331 3341 """ Upgrade your IPython installation
3332 3342
3333 3343 This will copy the config files that don't yet exist in your
3334 3344 ipython dir from the system config dir. Use this after upgrading
3335 3345 IPython if you don't wish to delete your .ipython dir.
3336 3346
3337 3347 Call with -nolegacy to get rid of ipythonrc* files (recommended for
3338 3348 new users)
3339 3349
3340 3350 """
3341 3351 ip = self.getapi()
3342 3352 ipinstallation = path(IPython.__file__).dirname()
3343 3353 upgrade_script = '%s "%s"' % (sys.executable,ipinstallation / 'upgrade_dir.py')
3344 3354 src_config = ipinstallation / 'UserConfig'
3345 3355 userdir = path(ip.options.ipythondir)
3346 3356 cmd = '%s "%s" "%s"' % (upgrade_script, src_config, userdir)
3347 3357 print ">",cmd
3348 3358 shell(cmd)
3349 3359 if arg == '-nolegacy':
3350 3360 legacy = userdir.files('ipythonrc*')
3351 3361 print "Nuking legacy files:",legacy
3352 3362
3353 3363 [p.remove() for p in legacy]
3354 3364 suffix = (sys.platform == 'win32' and '.ini' or '')
3355 3365 (userdir / ('ipythonrc' + suffix)).write_text('# Empty, see ipy_user_conf.py\n')
3356 3366
3357 3367
3358 3368 def magic_doctest_mode(self,parameter_s=''):
3359 3369 """Toggle doctest mode on and off.
3360 3370
3361 3371 This mode allows you to toggle the prompt behavior between normal
3362 3372 IPython prompts and ones that are as similar to the default IPython
3363 3373 interpreter as possible.
3364 3374
3365 3375 It also supports the pasting of code snippets that have leading '>>>'
3366 3376 and '...' prompts in them. This means that you can paste doctests from
3367 3377 files or docstrings (even if they have leading whitespace), and the
3368 3378 code will execute correctly. You can then use '%history -tn' to see
3369 3379 the translated history without line numbers; this will give you the
3370 3380 input after removal of all the leading prompts and whitespace, which
3371 3381 can be pasted back into an editor.
3372 3382
3373 3383 With these features, you can switch into this mode easily whenever you
3374 3384 need to do testing and changes to doctests, without having to leave
3375 3385 your existing IPython session.
3376 3386 """
3377 3387
3378 3388 # XXX - Fix this to have cleaner activate/deactivate calls.
3379 3389 from IPython.Extensions import InterpreterPasteInput as ipaste
3380 3390 from IPython.ipstruct import Struct
3381 3391
3382 3392 # Shorthands
3383 3393 shell = self.shell
3384 3394 oc = shell.outputcache
3385 3395 rc = shell.rc
3386 3396 meta = shell.meta
3387 3397 # dstore is a data store kept in the instance metadata bag to track any
3388 3398 # changes we make, so we can undo them later.
3389 3399 dstore = meta.setdefault('doctest_mode',Struct())
3390 3400 save_dstore = dstore.setdefault
3391 3401
3392 3402 # save a few values we'll need to recover later
3393 3403 mode = save_dstore('mode',False)
3394 3404 save_dstore('rc_pprint',rc.pprint)
3395 3405 save_dstore('xmode',shell.InteractiveTB.mode)
3396 3406 save_dstore('rc_separate_out',rc.separate_out)
3397 3407 save_dstore('rc_separate_out2',rc.separate_out2)
3398 3408 save_dstore('rc_prompts_pad_left',rc.prompts_pad_left)
3399 3409 save_dstore('rc_separate_in',rc.separate_in)
3400 3410
3401 3411 if mode == False:
3402 3412 # turn on
3403 3413 ipaste.activate_prefilter()
3404 3414
3405 3415 oc.prompt1.p_template = '>>> '
3406 3416 oc.prompt2.p_template = '... '
3407 3417 oc.prompt_out.p_template = ''
3408 3418
3409 3419 # Prompt separators like plain python
3410 3420 oc.input_sep = oc.prompt1.sep = ''
3411 3421 oc.output_sep = ''
3412 3422 oc.output_sep2 = ''
3413 3423
3414 3424 oc.prompt1.pad_left = oc.prompt2.pad_left = \
3415 3425 oc.prompt_out.pad_left = False
3416 3426
3417 3427 rc.pprint = False
3418 3428
3419 3429 shell.magic_xmode('Plain')
3420 3430
3421 3431 else:
3422 3432 # turn off
3423 3433 ipaste.deactivate_prefilter()
3424 3434
3425 3435 oc.prompt1.p_template = rc.prompt_in1
3426 3436 oc.prompt2.p_template = rc.prompt_in2
3427 3437 oc.prompt_out.p_template = rc.prompt_out
3428 3438
3429 3439 oc.input_sep = oc.prompt1.sep = dstore.rc_separate_in
3430 3440
3431 3441 oc.output_sep = dstore.rc_separate_out
3432 3442 oc.output_sep2 = dstore.rc_separate_out2
3433 3443
3434 3444 oc.prompt1.pad_left = oc.prompt2.pad_left = \
3435 3445 oc.prompt_out.pad_left = dstore.rc_prompts_pad_left
3436 3446
3437 3447 rc.pprint = dstore.rc_pprint
3438 3448
3439 3449 shell.magic_xmode(dstore.xmode)
3440 3450
3441 3451 # Store new mode and inform
3442 3452 dstore.mode = bool(1-int(mode))
3443 3453 print 'Doctest mode is:',
3444 3454 print ['OFF','ON'][dstore.mode]
3445 3455
3446 3456 # end Magic
@@ -1,622 +1,626 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Classes for handling input/output prompts.
4 4 """
5 5
6 6 #*****************************************************************************
7 7 # Copyright (C) 2008-2009 The IPython Development Team
8 8 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #*****************************************************************************
13 13
14 14 #****************************************************************************
15 15 # Required modules
16 16 import __builtin__
17 17 import os
18 18 import socket
19 19 import sys
20 20 import time
21 21
22 22 # IPython's own
23 23 from IPython import ColorANSI
24 24 from IPython import Release
25 25 from IPython.external.Itpl import ItplNS
26 26 from IPython.ipapi import TryNext
27 27 from IPython.ipstruct import Struct
28 28 from IPython.macro import Macro
29 29
30 30 from IPython.genutils import *
31 31
32 32 #****************************************************************************
33 33 #Color schemes for Prompts.
34 34
35 35 PromptColors = ColorANSI.ColorSchemeTable()
36 36 InputColors = ColorANSI.InputTermColors # just a shorthand
37 37 Colors = ColorANSI.TermColors # just a shorthand
38 38
39 39 PromptColors.add_scheme(ColorANSI.ColorScheme(
40 40 'NoColor',
41 41 in_prompt = InputColors.NoColor, # Input prompt
42 42 in_number = InputColors.NoColor, # Input prompt number
43 43 in_prompt2 = InputColors.NoColor, # Continuation prompt
44 44 in_normal = InputColors.NoColor, # color off (usu. Colors.Normal)
45 45
46 46 out_prompt = Colors.NoColor, # Output prompt
47 47 out_number = Colors.NoColor, # Output prompt number
48 48
49 49 normal = Colors.NoColor # color off (usu. Colors.Normal)
50 50 ))
51 51
52 52 # make some schemes as instances so we can copy them for modification easily:
53 53 __PColLinux = ColorANSI.ColorScheme(
54 54 'Linux',
55 55 in_prompt = InputColors.Green,
56 56 in_number = InputColors.LightGreen,
57 57 in_prompt2 = InputColors.Green,
58 58 in_normal = InputColors.Normal, # color off (usu. Colors.Normal)
59 59
60 60 out_prompt = Colors.Red,
61 61 out_number = Colors.LightRed,
62 62
63 63 normal = Colors.Normal
64 64 )
65 65 # Don't forget to enter it into the table!
66 66 PromptColors.add_scheme(__PColLinux)
67 67
68 68 # Slightly modified Linux for light backgrounds
69 69 __PColLightBG = __PColLinux.copy('LightBG')
70 70
71 71 __PColLightBG.colors.update(
72 72 in_prompt = InputColors.Blue,
73 73 in_number = InputColors.LightBlue,
74 74 in_prompt2 = InputColors.Blue
75 75 )
76 76 PromptColors.add_scheme(__PColLightBG)
77 77
78 78 del Colors,InputColors
79 79
80 80 #-----------------------------------------------------------------------------
81 81 def multiple_replace(dict, text):
82 82 """ Replace in 'text' all occurences of any key in the given
83 83 dictionary by its corresponding value. Returns the new string."""
84 84
85 85 # Function by Xavier Defrang, originally found at:
86 86 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330
87 87
88 88 # Create a regular expression from the dictionary keys
89 89 regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
90 90 # For each match, look-up corresponding value in dictionary
91 91 return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Special characters that can be used in prompt templates, mainly bash-like
95 95
96 96 # If $HOME isn't defined (Windows), make it an absurd string so that it can
97 97 # never be expanded out into '~'. Basically anything which can never be a
98 98 # reasonable directory name will do, we just want the $HOME -> '~' operation
99 99 # to become a no-op. We pre-compute $HOME here so it's not done on every
100 100 # prompt call.
101 101
102 102 # FIXME:
103 103
104 104 # - This should be turned into a class which does proper namespace management,
105 105 # since the prompt specials need to be evaluated in a certain namespace.
106 106 # Currently it's just globals, which need to be managed manually by code
107 107 # below.
108 108
109 109 # - I also need to split up the color schemes from the prompt specials
110 110 # somehow. I don't have a clean design for that quite yet.
111 111
112 112 HOME = os.environ.get("HOME","//////:::::ZZZZZ,,,~~~")
113 113
114 114 # We precompute a few more strings here for the prompt_specials, which are
115 115 # fixed once ipython starts. This reduces the runtime overhead of computing
116 116 # prompt strings.
117 117 USER = os.environ.get("USER")
118 118 HOSTNAME = socket.gethostname()
119 119 HOSTNAME_SHORT = HOSTNAME.split(".")[0]
120 120 ROOT_SYMBOL = "$#"[os.name=='nt' or os.getuid()==0]
121 121
122 122 prompt_specials_color = {
123 123 # Prompt/history count
124 124 '%n' : '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
125 125 r'\#': '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
126 126 # Just the prompt counter number, WITHOUT any coloring wrappers, so users
127 127 # can get numbers displayed in whatever color they want.
128 128 r'\N': '${self.cache.prompt_count}',
129
129 130 # Prompt/history count, with the actual digits replaced by dots. Used
130 131 # mainly in continuation prompts (prompt_in2)
132 #r'\D': '${"."*len(str(self.cache.prompt_count))}',
133 # More robust form of the above expression, that uses __builtins__
131 134 r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}',
135
132 136 # Current working directory
133 137 r'\w': '${os.getcwd()}',
134 138 # Current time
135 139 r'\t' : '${time.strftime("%H:%M:%S")}',
136 140 # Basename of current working directory.
137 141 # (use os.sep to make this portable across OSes)
138 142 r'\W' : '${os.getcwd().split("%s")[-1]}' % os.sep,
139 143 # These X<N> are an extension to the normal bash prompts. They return
140 144 # N terms of the path, after replacing $HOME with '~'
141 145 r'\X0': '${os.getcwd().replace("%s","~")}' % HOME,
142 146 r'\X1': '${self.cwd_filt(1)}',
143 147 r'\X2': '${self.cwd_filt(2)}',
144 148 r'\X3': '${self.cwd_filt(3)}',
145 149 r'\X4': '${self.cwd_filt(4)}',
146 150 r'\X5': '${self.cwd_filt(5)}',
147 151 # Y<N> are similar to X<N>, but they show '~' if it's the directory
148 152 # N+1 in the list. Somewhat like %cN in tcsh.
149 153 r'\Y0': '${self.cwd_filt2(0)}',
150 154 r'\Y1': '${self.cwd_filt2(1)}',
151 155 r'\Y2': '${self.cwd_filt2(2)}',
152 156 r'\Y3': '${self.cwd_filt2(3)}',
153 157 r'\Y4': '${self.cwd_filt2(4)}',
154 158 r'\Y5': '${self.cwd_filt2(5)}',
155 159 # Hostname up to first .
156 160 r'\h': HOSTNAME_SHORT,
157 161 # Full hostname
158 162 r'\H': HOSTNAME,
159 163 # Username of current user
160 164 r'\u': USER,
161 165 # Escaped '\'
162 166 '\\\\': '\\',
163 167 # Newline
164 168 r'\n': '\n',
165 169 # Carriage return
166 170 r'\r': '\r',
167 171 # Release version
168 172 r'\v': Release.version,
169 173 # Root symbol ($ or #)
170 174 r'\$': ROOT_SYMBOL,
171 175 }
172 176
173 177 # A copy of the prompt_specials dictionary but with all color escapes removed,
174 178 # so we can correctly compute the prompt length for the auto_rewrite method.
175 179 prompt_specials_nocolor = prompt_specials_color.copy()
176 180 prompt_specials_nocolor['%n'] = '${self.cache.prompt_count}'
177 181 prompt_specials_nocolor[r'\#'] = '${self.cache.prompt_count}'
178 182
179 183 # Add in all the InputTermColors color escapes as valid prompt characters.
180 184 # They all get added as \\C_COLORNAME, so that we don't have any conflicts
181 185 # with a color name which may begin with a letter used by any other of the
182 186 # allowed specials. This of course means that \\C will never be allowed for
183 187 # anything else.
184 188 input_colors = ColorANSI.InputTermColors
185 189 for _color in dir(input_colors):
186 190 if _color[0] != '_':
187 191 c_name = r'\C_'+_color
188 192 prompt_specials_color[c_name] = getattr(input_colors,_color)
189 193 prompt_specials_nocolor[c_name] = ''
190 194
191 195 # we default to no color for safety. Note that prompt_specials is a global
192 196 # variable used by all prompt objects.
193 197 prompt_specials = prompt_specials_nocolor
194 198
195 199 #-----------------------------------------------------------------------------
196 200 def str_safe(arg):
197 201 """Convert to a string, without ever raising an exception.
198 202
199 203 If str(arg) fails, <ERROR: ... > is returned, where ... is the exception
200 204 error message."""
201 205
202 206 try:
203 207 out = str(arg)
204 208 except UnicodeError:
205 209 try:
206 210 out = arg.encode('utf_8','replace')
207 211 except Exception,msg:
208 212 # let's keep this little duplication here, so that the most common
209 213 # case doesn't suffer from a double try wrapping.
210 214 out = '<ERROR: %s>' % msg
211 215 except Exception,msg:
212 216 out = '<ERROR: %s>' % msg
213 217 return out
214 218
215 219 class BasePrompt(object):
216 220 """Interactive prompt similar to Mathematica's."""
217 221
218 222 def _get_p_template(self):
219 223 return self._p_template
220 224
221 225 def _set_p_template(self,val):
222 226 self._p_template = val
223 227 self.set_p_str()
224 228
225 229 p_template = property(_get_p_template,_set_p_template,
226 230 doc='Template for prompt string creation')
227 231
228 232 def __init__(self,cache,sep,prompt,pad_left=False):
229 233
230 234 # Hack: we access information about the primary prompt through the
231 235 # cache argument. We need this, because we want the secondary prompt
232 236 # to be aligned with the primary one. Color table info is also shared
233 237 # by all prompt classes through the cache. Nice OO spaghetti code!
234 238 self.cache = cache
235 239 self.sep = sep
236 240
237 241 # regexp to count the number of spaces at the end of a prompt
238 242 # expression, useful for prompt auto-rewriting
239 243 self.rspace = re.compile(r'(\s*)$')
240 244 # Flag to left-pad prompt strings to match the length of the primary
241 245 # prompt
242 246 self.pad_left = pad_left
243 247
244 248 # Set template to create each actual prompt (where numbers change).
245 249 # Use a property
246 250 self.p_template = prompt
247 251 self.set_p_str()
248 252
249 253 def set_p_str(self):
250 254 """ Set the interpolating prompt strings.
251 255
252 256 This must be called every time the color settings change, because the
253 257 prompt_specials global may have changed."""
254 258
255 259 import os,time # needed in locals for prompt string handling
256 260 loc = locals()
257 261 try:
258 262 self.p_str = ItplNS('%s%s%s' %
259 263 ('${self.sep}${self.col_p}',
260 264 multiple_replace(prompt_specials, self.p_template),
261 265 '${self.col_norm}'),self.cache.user_ns,loc)
262 266
263 267 self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
264 268 self.p_template),
265 269 self.cache.user_ns,loc)
266 270 except:
267 271 print "Illegal prompt template (check $ usage!):",self.p_template
268 272 self.p_str = self.p_template
269 273 self.p_str_nocolor = self.p_template
270 274
271 275 def write(self,msg): # dbg
272 276 sys.stdout.write(msg)
273 277 return ''
274 278
275 279 def __str__(self):
276 280 """Return a string form of the prompt.
277 281
278 282 This for is useful for continuation and output prompts, since it is
279 283 left-padded to match lengths with the primary one (if the
280 284 self.pad_left attribute is set)."""
281 285
282 286 out_str = str_safe(self.p_str)
283 287 if self.pad_left:
284 288 # We must find the amount of padding required to match lengths,
285 289 # taking the color escapes (which are invisible on-screen) into
286 290 # account.
287 291 esc_pad = len(out_str) - len(str_safe(self.p_str_nocolor))
288 292 format = '%%%ss' % (len(str(self.cache.last_prompt))+esc_pad)
289 293 return format % out_str
290 294 else:
291 295 return out_str
292 296
293 297 # these path filters are put in as methods so that we can control the
294 298 # namespace where the prompt strings get evaluated
295 299 def cwd_filt(self,depth):
296 300 """Return the last depth elements of the current working directory.
297 301
298 302 $HOME is always replaced with '~'.
299 303 If depth==0, the full path is returned."""
300 304
301 305 cwd = os.getcwd().replace(HOME,"~")
302 306 out = os.sep.join(cwd.split(os.sep)[-depth:])
303 307 if out:
304 308 return out
305 309 else:
306 310 return os.sep
307 311
308 312 def cwd_filt2(self,depth):
309 313 """Return the last depth elements of the current working directory.
310 314
311 315 $HOME is always replaced with '~'.
312 316 If depth==0, the full path is returned."""
313 317
314 318 full_cwd = os.getcwd()
315 319 cwd = full_cwd.replace(HOME,"~").split(os.sep)
316 320 if '~' in cwd and len(cwd) == depth+1:
317 321 depth += 1
318 322 drivepart = ''
319 323 if sys.platform == 'win32' and len(cwd) > depth:
320 324 drivepart = os.path.splitdrive(full_cwd)[0]
321 325 out = drivepart + '/'.join(cwd[-depth:])
322 326
323 327 if out:
324 328 return out
325 329 else:
326 330 return os.sep
327 331
328 332 def __nonzero__(self):
329 333 """Implement boolean behavior.
330 334
331 335 Checks whether the p_str attribute is non-empty"""
332 336
333 337 return bool(self.p_template)
334 338
335 339 class Prompt1(BasePrompt):
336 340 """Input interactive prompt similar to Mathematica's."""
337 341
338 342 def __init__(self,cache,sep='\n',prompt='In [\\#]: ',pad_left=True):
339 343 BasePrompt.__init__(self,cache,sep,prompt,pad_left)
340 344
341 345 def set_colors(self):
342 346 self.set_p_str()
343 347 Colors = self.cache.color_table.active_colors # shorthand
344 348 self.col_p = Colors.in_prompt
345 349 self.col_num = Colors.in_number
346 350 self.col_norm = Colors.in_normal
347 351 # We need a non-input version of these escapes for the '--->'
348 352 # auto-call prompts used in the auto_rewrite() method.
349 353 self.col_p_ni = self.col_p.replace('\001','').replace('\002','')
350 354 self.col_norm_ni = Colors.normal
351 355
352 356 def __str__(self):
353 357 self.cache.prompt_count += 1
354 358 self.cache.last_prompt = str_safe(self.p_str_nocolor).split('\n')[-1]
355 359 return str_safe(self.p_str)
356 360
357 361 def auto_rewrite(self):
358 362 """Print a string of the form '--->' which lines up with the previous
359 363 input string. Useful for systems which re-write the user input when
360 364 handling automatically special syntaxes."""
361 365
362 366 curr = str(self.cache.last_prompt)
363 367 nrspaces = len(self.rspace.search(curr).group())
364 368 return '%s%s>%s%s' % (self.col_p_ni,'-'*(len(curr)-nrspaces-1),
365 369 ' '*nrspaces,self.col_norm_ni)
366 370
367 371 class PromptOut(BasePrompt):
368 372 """Output interactive prompt similar to Mathematica's."""
369 373
370 374 def __init__(self,cache,sep='',prompt='Out[\\#]: ',pad_left=True):
371 375 BasePrompt.__init__(self,cache,sep,prompt,pad_left)
372 376 if not self.p_template:
373 377 self.__str__ = lambda: ''
374 378
375 379 def set_colors(self):
376 380 self.set_p_str()
377 381 Colors = self.cache.color_table.active_colors # shorthand
378 382 self.col_p = Colors.out_prompt
379 383 self.col_num = Colors.out_number
380 384 self.col_norm = Colors.normal
381 385
382 386 class Prompt2(BasePrompt):
383 387 """Interactive continuation prompt."""
384 388
385 389 def __init__(self,cache,prompt=' .\\D.: ',pad_left=True):
386 390 self.cache = cache
387 391 self.p_template = prompt
388 392 self.pad_left = pad_left
389 393 self.set_p_str()
390 394
391 395 def set_p_str(self):
392 396 import os,time # needed in locals for prompt string handling
393 397 loc = locals()
394 398 self.p_str = ItplNS('%s%s%s' %
395 399 ('${self.col_p2}',
396 400 multiple_replace(prompt_specials, self.p_template),
397 401 '$self.col_norm'),
398 402 self.cache.user_ns,loc)
399 403 self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
400 404 self.p_template),
401 405 self.cache.user_ns,loc)
402 406
403 407 def set_colors(self):
404 408 self.set_p_str()
405 409 Colors = self.cache.color_table.active_colors
406 410 self.col_p2 = Colors.in_prompt2
407 411 self.col_norm = Colors.in_normal
408 412 # FIXME (2004-06-16) HACK: prevent crashes for users who haven't
409 413 # updated their prompt_in2 definitions. Remove eventually.
410 414 self.col_p = Colors.out_prompt
411 415 self.col_num = Colors.out_number
412 416
413 417
414 418 #-----------------------------------------------------------------------------
415 419 class CachedOutput:
416 420 """Class for printing output from calculations while keeping a cache of
417 421 reults. It dynamically creates global variables prefixed with _ which
418 422 contain these results.
419 423
420 424 Meant to be used as a sys.displayhook replacement, providing numbered
421 425 prompts and cache services.
422 426
423 427 Initialize with initial and final values for cache counter (this defines
424 428 the maximum size of the cache."""
425 429
426 430 def __init__(self,shell,cache_size,Pprint,
427 431 colors='NoColor',input_sep='\n',
428 432 output_sep='\n',output_sep2='',
429 433 ps1 = None, ps2 = None,ps_out = None,pad_left=True):
430 434
431 435 cache_size_min = 3
432 436 if cache_size <= 0:
433 437 self.do_full_cache = 0
434 438 cache_size = 0
435 439 elif cache_size < cache_size_min:
436 440 self.do_full_cache = 0
437 441 cache_size = 0
438 442 warn('caching was disabled (min value for cache size is %s).' %
439 443 cache_size_min,level=3)
440 444 else:
441 445 self.do_full_cache = 1
442 446
443 447 self.cache_size = cache_size
444 448 self.input_sep = input_sep
445 449
446 450 # we need a reference to the user-level namespace
447 451 self.shell = shell
448 452 self.user_ns = shell.user_ns
449 453 # and to the user's input
450 454 self.input_hist = shell.input_hist
451 455 # and to the user's logger, for logging output
452 456 self.logger = shell.logger
453 457
454 458 # Set input prompt strings and colors
455 459 if cache_size == 0:
456 460 if ps1.find('%n') > -1 or ps1.find(r'\#') > -1 \
457 461 or ps1.find(r'\N') > -1:
458 462 ps1 = '>>> '
459 463 if ps2.find('%n') > -1 or ps2.find(r'\#') > -1 \
460 464 or ps2.find(r'\N') > -1:
461 465 ps2 = '... '
462 466 self.ps1_str = self._set_prompt_str(ps1,'In [\\#]: ','>>> ')
463 467 self.ps2_str = self._set_prompt_str(ps2,' .\\D.: ','... ')
464 468 self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','')
465 469
466 470 self.color_table = PromptColors
467 471 self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str,
468 472 pad_left=pad_left)
469 473 self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left)
470 474 self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str,
471 475 pad_left=pad_left)
472 476 self.set_colors(colors)
473 477
474 478 # other more normal stuff
475 479 # b/c each call to the In[] prompt raises it by 1, even the first.
476 480 self.prompt_count = 0
477 481 # Store the last prompt string each time, we need it for aligning
478 482 # continuation and auto-rewrite prompts
479 483 self.last_prompt = ''
480 484 self.Pprint = Pprint
481 485 self.output_sep = output_sep
482 486 self.output_sep2 = output_sep2
483 487 self._,self.__,self.___ = '','',''
484 488 self.pprint_types = map(type,[(),[],{}])
485 489
486 490 # these are deliberately global:
487 491 to_user_ns = {'_':self._,'__':self.__,'___':self.___}
488 492 self.user_ns.update(to_user_ns)
489 493
490 494 def _set_prompt_str(self,p_str,cache_def,no_cache_def):
491 495 if p_str is None:
492 496 if self.do_full_cache:
493 497 return cache_def
494 498 else:
495 499 return no_cache_def
496 500 else:
497 501 return p_str
498 502
499 503 def set_colors(self,colors):
500 504 """Set the active color scheme and configure colors for the three
501 505 prompt subsystems."""
502 506
503 507 # FIXME: the prompt_specials global should be gobbled inside this
504 508 # class instead. Do it when cleaning up the whole 3-prompt system.
505 509 global prompt_specials
506 510 if colors.lower()=='nocolor':
507 511 prompt_specials = prompt_specials_nocolor
508 512 else:
509 513 prompt_specials = prompt_specials_color
510 514
511 515 self.color_table.set_active_scheme(colors)
512 516 self.prompt1.set_colors()
513 517 self.prompt2.set_colors()
514 518 self.prompt_out.set_colors()
515 519
516 520 def __call__(self,arg=None):
517 521 """Printing with history cache management.
518 522
519 523 This is invoked everytime the interpreter needs to print, and is
520 524 activated by setting the variable sys.displayhook to it."""
521 525
522 526 # If something injected a '_' variable in __builtin__, delete
523 527 # ipython's automatic one so we don't clobber that. gettext() in
524 528 # particular uses _, so we need to stay away from it.
525 529 if '_' in __builtin__.__dict__:
526 530 try:
527 531 del self.user_ns['_']
528 532 except KeyError:
529 533 pass
530 534 if arg is not None:
531 535 cout_write = Term.cout.write # fast lookup
532 536 # first handle the cache and counters
533 537
534 538 # do not print output if input ends in ';'
535 539 try:
536 540 if self.input_hist[self.prompt_count].endswith(';\n'):
537 541 return
538 542 except IndexError:
539 543 # some uses of ipshellembed may fail here
540 544 pass
541 545 # don't use print, puts an extra space
542 546 cout_write(self.output_sep)
543 547 outprompt = self.shell.hooks.generate_output_prompt()
544 548 if self.do_full_cache:
545 549 cout_write(outprompt)
546 550
547 551 # and now call a possibly user-defined print mechanism
548 552 manipulated_val = self.display(arg)
549 553
550 554 # user display hooks can change the variable to be stored in
551 555 # output history
552 556
553 557 if manipulated_val is not None:
554 558 arg = manipulated_val
555 559
556 560 # avoid recursive reference when displaying _oh/Out
557 561 if arg is not self.user_ns['_oh']:
558 562 self.update(arg)
559 563
560 564 if self.logger.log_output:
561 565 self.logger.log_write(repr(arg),'output')
562 566 cout_write(self.output_sep2)
563 567 Term.cout.flush()
564 568
565 569 def _display(self,arg):
566 570 """Default printer method, uses pprint.
567 571
568 572 Do ip.set_hook("result_display", my_displayhook) for custom result
569 573 display, e.g. when your own objects need special formatting.
570 574 """
571 575 try:
572 576 return IPython.generics.result_display(arg)
573 577 except TryNext:
574 578 return self.shell.hooks.result_display(arg)
575 579
576 580 # Assign the default display method:
577 581 display = _display
578 582
579 583 def update(self,arg):
580 584 #print '***cache_count', self.cache_count # dbg
581 585 if len(self.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
582 586 warn('Output cache limit (currently '+
583 587 `self.cache_size`+' entries) hit.\n'
584 588 'Flushing cache and resetting history counter...\n'
585 589 'The only history variables available will be _,__,___ and _1\n'
586 590 'with the current result.')
587 591
588 592 self.flush()
589 593 # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
590 594 # we cause buggy behavior for things like gettext).
591 595 if '_' not in __builtin__.__dict__:
592 596 self.___ = self.__
593 597 self.__ = self._
594 598 self._ = arg
595 599 self.user_ns.update({'_':self._,'__':self.__,'___':self.___})
596 600
597 601 # hackish access to top-level namespace to create _1,_2... dynamically
598 602 to_main = {}
599 603 if self.do_full_cache:
600 604 new_result = '_'+`self.prompt_count`
601 605 to_main[new_result] = arg
602 606 self.user_ns.update(to_main)
603 607 self.user_ns['_oh'][self.prompt_count] = arg
604 608
605 609 def flush(self):
606 610 if not self.do_full_cache:
607 611 raise ValueError,"You shouldn't have reached the cache flush "\
608 612 "if full caching is not enabled!"
609 613 # delete auto-generated vars from global namespace
610 614
611 615 for n in range(1,self.prompt_count + 1):
612 616 key = '_'+`n`
613 617 try:
614 618 del self.user_ns[key]
615 619 except: pass
616 620 self.user_ns['_oh'].clear()
617 621
618 622 if '_' not in __builtin__.__dict__:
619 623 self.user_ns.update({'_':None,'__':None, '___':None})
620 624 import gc
621 625 gc.collect() # xxx needed?
622 626
@@ -1,123 +1,125 b''
1 1 # encoding: utf-8
2 2
3 3 """The IPython Core Notification Center.
4 4
5 5 See docs/source/development/notification_blueprint.txt for an overview of the
6 6 notification module.
7 7 """
8 8
9 9 __docformat__ = "restructuredtext en"
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2008 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 # Tell nose to skip the testing of this module
19 __test__ = {}
18 20
19 21 class NotificationCenter(object):
20 22 """Synchronous notification center
21 23
22 24 Example
23 25 -------
24 26 >>> import IPython.kernel.core.notification as notification
25 27 >>> def callback(theType, theSender, args={}):
26 28 ... print theType,theSender,args
27 29 ...
28 30 >>> notification.sharedCenter.add_observer(callback, 'NOTIFICATION_TYPE', None)
29 31 >>> notification.sharedCenter.post_notification('NOTIFICATION_TYPE', object()) # doctest:+ELLIPSIS
30 32 NOTIFICATION_TYPE ...
31 33
32 34 """
33 35 def __init__(self):
34 36 super(NotificationCenter, self).__init__()
35 37 self._init_observers()
36 38
37 39
38 40 def _init_observers(self):
39 41 """Initialize observer storage"""
40 42
41 43 self.registered_types = set() #set of types that are observed
42 44 self.registered_senders = set() #set of senders that are observed
43 45 self.observers = {} #map (type,sender) => callback (callable)
44 46
45 47
46 48 def post_notification(self, theType, sender, **kwargs):
47 49 """Post notification (type,sender,**kwargs) to all registered
48 50 observers.
49 51
50 52 Implementation
51 53 --------------
52 54 * If no registered observers, performance is O(1).
53 55 * Notificaiton order is undefined.
54 56 * Notifications are posted synchronously.
55 57 """
56 58
57 59 if(theType==None or sender==None):
58 60 raise Exception("NotificationCenter.post_notification requires \
59 61 type and sender.")
60 62
61 63 # If there are no registered observers for the type/sender pair
62 64 if((theType not in self.registered_types and
63 65 None not in self.registered_types) or
64 66 (sender not in self.registered_senders and
65 67 None not in self.registered_senders)):
66 68 return
67 69
68 70 for o in self._observers_for_notification(theType, sender):
69 71 o(theType, sender, args=kwargs)
70 72
71 73
72 74 def _observers_for_notification(self, theType, sender):
73 75 """Find all registered observers that should recieve notification"""
74 76
75 77 keys = (
76 78 (theType,sender),
77 79 (theType, None),
78 80 (None, sender),
79 81 (None,None)
80 82 )
81 83
82 84
83 85 obs = set()
84 86 for k in keys:
85 87 obs.update(self.observers.get(k, set()))
86 88
87 89 return obs
88 90
89 91
90 92 def add_observer(self, callback, theType, sender):
91 93 """Add an observer callback to this notification center.
92 94
93 95 The given callback will be called upon posting of notifications of
94 96 the given type/sender and will receive any additional kwargs passed
95 97 to post_notification.
96 98
97 99 Parameters
98 100 ----------
99 101 observerCallback : callable
100 102 Callable. Must take at least two arguments::
101 103 observerCallback(type, sender, args={})
102 104
103 105 theType : hashable
104 106 The notification type. If None, all notifications from sender
105 107 will be posted.
106 108
107 109 sender : hashable
108 110 The notification sender. If None, all notifications of theType
109 111 will be posted.
110 112 """
111 113 assert(callback != None)
112 114 self.registered_types.add(theType)
113 115 self.registered_senders.add(sender)
114 116 self.observers.setdefault((theType,sender), set()).add(callback)
115 117
116 118 def remove_all_observers(self):
117 119 """Removes all observers from this notification center"""
118 120
119 121 self._init_observers()
120 122
121 123
122 124
123 125 sharedCenter = NotificationCenter() No newline at end of file
@@ -1,903 +1,906 b''
1 1 # encoding: utf-8
2 2 # -*- test-case-name: IPython.kernel.tests.test_engineservice -*-
3 3
4 4 """A Twisted Service Representation of the IPython core.
5 5
6 6 The IPython Core exposed to the network is called the Engine. Its
7 7 representation in Twisted in the EngineService. Interfaces and adapters
8 8 are used to abstract out the details of the actual network protocol used.
9 9 The EngineService is an Engine that knows nothing about the actual protocol
10 10 used.
11 11
12 12 The EngineService is exposed with various network protocols in modules like:
13 13
14 14 enginepb.py
15 15 enginevanilla.py
16 16
17 17 As of 12/12/06 the classes in this module have been simplified greatly. It was
18 18 felt that we had over-engineered things. To improve the maintainability of the
19 19 code we have taken out the ICompleteEngine interface and the completeEngine
20 20 method that automatically added methods to engines.
21 21
22 22 """
23 23
24 24 __docformat__ = "restructuredtext en"
25 25
26 26 #-------------------------------------------------------------------------------
27 27 # Copyright (C) 2008 The IPython Development Team
28 28 #
29 29 # Distributed under the terms of the BSD License. The full license is in
30 30 # the file COPYING, distributed as part of this software.
31 31 #-------------------------------------------------------------------------------
32 32
33 33 #-------------------------------------------------------------------------------
34 34 # Imports
35 35 #-------------------------------------------------------------------------------
36 36
37 # Tell nose to skip the testing of this module
38 __test__ = {}
39
37 40 import os, sys, copy
38 41 import cPickle as pickle
39 42 from new import instancemethod
40 43
41 44 from twisted.application import service
42 45 from twisted.internet import defer, reactor
43 46 from twisted.python import log, failure, components
44 47 import zope.interface as zi
45 48
46 49 from IPython.kernel.core.interpreter import Interpreter
47 50 from IPython.kernel import newserialized, error, util
48 51 from IPython.kernel.util import printer
49 52 from IPython.kernel.twistedutil import gatherBoth, DeferredList
50 53 from IPython.kernel import codeutil
51 54
52 55
53 56 #-------------------------------------------------------------------------------
54 57 # Interface specification for the Engine
55 58 #-------------------------------------------------------------------------------
56 59
57 60 class IEngineCore(zi.Interface):
58 61 """The minimal required interface for the IPython Engine.
59 62
60 63 This interface provides a formal specification of the IPython core.
61 64 All these methods should return deferreds regardless of what side of a
62 65 network connection they are on.
63 66
64 67 In general, this class simply wraps a shell class and wraps its return
65 68 values as Deferred objects. If the underlying shell class method raises
66 69 an exception, this class should convert it to a twisted.failure.Failure
67 70 that will be propagated along the Deferred's errback chain.
68 71
69 72 In addition, Failures are aggressive. By this, we mean that if a method
70 73 is performing multiple actions (like pulling multiple object) if any
71 74 single one fails, the entire method will fail with that Failure. It is
72 75 all or nothing.
73 76 """
74 77
75 78 id = zi.interface.Attribute("the id of the Engine object")
76 79 properties = zi.interface.Attribute("A dict of properties of the Engine")
77 80
78 81 def execute(lines):
79 82 """Execute lines of Python code.
80 83
81 84 Returns a dictionary with keys (id, number, stdin, stdout, stderr)
82 85 upon success.
83 86
84 87 Returns a failure object if the execution of lines raises an exception.
85 88 """
86 89
87 90 def push(namespace):
88 91 """Push dict namespace into the user's namespace.
89 92
90 93 Returns a deferred to None or a failure.
91 94 """
92 95
93 96 def pull(keys):
94 97 """Pulls values out of the user's namespace by keys.
95 98
96 99 Returns a deferred to a tuple objects or a single object.
97 100
98 101 Raises NameError if any one of objects doess not exist.
99 102 """
100 103
101 104 def push_function(namespace):
102 105 """Push a dict of key, function pairs into the user's namespace.
103 106
104 107 Returns a deferred to None or a failure."""
105 108
106 109 def pull_function(keys):
107 110 """Pulls functions out of the user's namespace by keys.
108 111
109 112 Returns a deferred to a tuple of functions or a single function.
110 113
111 114 Raises NameError if any one of the functions does not exist.
112 115 """
113 116
114 117 def get_result(i=None):
115 118 """Get the stdin/stdout/stderr of command i.
116 119
117 120 Returns a deferred to a dict with keys
118 121 (id, number, stdin, stdout, stderr).
119 122
120 123 Raises IndexError if command i does not exist.
121 124 Raises TypeError if i in not an int.
122 125 """
123 126
124 127 def reset():
125 128 """Reset the shell.
126 129
127 130 This clears the users namespace. Won't cause modules to be
128 131 reloaded. Should also re-initialize certain variables like id.
129 132 """
130 133
131 134 def kill():
132 135 """Kill the engine by stopping the reactor."""
133 136
134 137 def keys():
135 138 """Return the top level variables in the users namspace.
136 139
137 140 Returns a deferred to a dict."""
138 141
139 142
140 143 class IEngineSerialized(zi.Interface):
141 144 """Push/Pull methods that take Serialized objects.
142 145
143 146 All methods should return deferreds.
144 147 """
145 148
146 149 def push_serialized(namespace):
147 150 """Push a dict of keys and Serialized objects into the user's namespace."""
148 151
149 152 def pull_serialized(keys):
150 153 """Pull objects by key from the user's namespace as Serialized.
151 154
152 155 Returns a list of or one Serialized.
153 156
154 157 Raises NameError is any one of the objects does not exist.
155 158 """
156 159
157 160
158 161 class IEngineProperties(zi.Interface):
159 162 """Methods for access to the properties object of an Engine"""
160 163
161 164 properties = zi.Attribute("A StrictDict object, containing the properties")
162 165
163 166 def set_properties(properties):
164 167 """set properties by key and value"""
165 168
166 169 def get_properties(keys=None):
167 170 """get a list of properties by `keys`, if no keys specified, get all"""
168 171
169 172 def del_properties(keys):
170 173 """delete properties by `keys`"""
171 174
172 175 def has_properties(keys):
173 176 """get a list of bool values for whether `properties` has `keys`"""
174 177
175 178 def clear_properties():
176 179 """clear the properties dict"""
177 180
178 181 class IEngineBase(IEngineCore, IEngineSerialized, IEngineProperties):
179 182 """The basic engine interface that EngineService will implement.
180 183
181 184 This exists so it is easy to specify adapters that adapt to and from the
182 185 API that the basic EngineService implements.
183 186 """
184 187 pass
185 188
186 189 class IEngineQueued(IEngineBase):
187 190 """Interface for adding a queue to an IEngineBase.
188 191
189 192 This interface extends the IEngineBase interface to add methods for managing
190 193 the engine's queue. The implicit details of this interface are that the
191 194 execution of all methods declared in IEngineBase should appropriately be
192 195 put through a queue before execution.
193 196
194 197 All methods should return deferreds.
195 198 """
196 199
197 200 def clear_queue():
198 201 """Clear the queue."""
199 202
200 203 def queue_status():
201 204 """Get the queued and pending commands in the queue."""
202 205
203 206 def register_failure_observer(obs):
204 207 """Register an observer of pending Failures.
205 208
206 209 The observer must implement IFailureObserver.
207 210 """
208 211
209 212 def unregister_failure_observer(obs):
210 213 """Unregister an observer of pending Failures."""
211 214
212 215
213 216 class IEngineThreaded(zi.Interface):
214 217 """A place holder for threaded commands.
215 218
216 219 All methods should return deferreds.
217 220 """
218 221 pass
219 222
220 223
221 224 #-------------------------------------------------------------------------------
222 225 # Functions and classes to implement the EngineService
223 226 #-------------------------------------------------------------------------------
224 227
225 228
226 229 class StrictDict(dict):
227 230 """This is a strict copying dictionary for use as the interface to the
228 231 properties of an Engine.
229 232
230 233 :IMPORTANT:
231 234 This object copies the values you set to it, and returns copies to you
232 235 when you request them. The only way to change properties os explicitly
233 236 through the setitem and getitem of the dictionary interface.
234 237
235 238 Example:
236 239 >>> e = get_engine(id)
237 240 >>> L = [1,2,3]
238 241 >>> e.properties['L'] = L
239 242 >>> L == e.properties['L']
240 243 True
241 244 >>> L.append(99)
242 245 >>> L == e.properties['L']
243 246 False
244 247
245 248 Note that getitem copies, so calls to methods of objects do not affect
246 249 the properties, as seen here:
247 250
248 251 >>> e.properties[1] = range(2)
249 252 >>> print e.properties[1]
250 253 [0, 1]
251 254 >>> e.properties[1].append(2)
252 255 >>> print e.properties[1]
253 256 [0, 1]
254 257 """
255 258 def __init__(self, *args, **kwargs):
256 259 dict.__init__(self, *args, **kwargs)
257 260 self.modified = True
258 261
259 262 def __getitem__(self, key):
260 263 return copy.deepcopy(dict.__getitem__(self, key))
261 264
262 265 def __setitem__(self, key, value):
263 266 # check if this entry is valid for transport around the network
264 267 # and copying
265 268 try:
266 269 pickle.dumps(key, 2)
267 270 pickle.dumps(value, 2)
268 271 newvalue = copy.deepcopy(value)
269 272 except:
270 273 raise error.InvalidProperty(value)
271 274 dict.__setitem__(self, key, newvalue)
272 275 self.modified = True
273 276
274 277 def __delitem__(self, key):
275 278 dict.__delitem__(self, key)
276 279 self.modified = True
277 280
278 281 def update(self, dikt):
279 282 for k,v in dikt.iteritems():
280 283 self[k] = v
281 284
282 285 def pop(self, key):
283 286 self.modified = True
284 287 return dict.pop(self, key)
285 288
286 289 def popitem(self):
287 290 self.modified = True
288 291 return dict.popitem(self)
289 292
290 293 def clear(self):
291 294 self.modified = True
292 295 dict.clear(self)
293 296
294 297 def subDict(self, *keys):
295 298 d = {}
296 299 for key in keys:
297 300 d[key] = self[key]
298 301 return d
299 302
300 303
301 304
302 305 class EngineAPI(object):
303 306 """This is the object through which the user can edit the `properties`
304 307 attribute of an Engine.
305 308 The Engine Properties object copies all object in and out of itself.
306 309 See the EngineProperties object for details.
307 310 """
308 311 _fix=False
309 312 def __init__(self, id):
310 313 self.id = id
311 314 self.properties = StrictDict()
312 315 self._fix=True
313 316
314 317 def __setattr__(self, k,v):
315 318 if self._fix:
316 319 raise error.KernelError("I am protected!")
317 320 else:
318 321 object.__setattr__(self, k, v)
319 322
320 323 def __delattr__(self, key):
321 324 raise error.KernelError("I am protected!")
322 325
323 326
324 327 _apiDict = {}
325 328
326 329 def get_engine(id):
327 330 """Get the Engine API object, whcih currently just provides the properties
328 331 object, by ID"""
329 332 global _apiDict
330 333 if not _apiDict.get(id):
331 334 _apiDict[id] = EngineAPI(id)
332 335 return _apiDict[id]
333 336
334 337 def drop_engine(id):
335 338 """remove an engine"""
336 339 global _apiDict
337 340 if _apiDict.has_key(id):
338 341 del _apiDict[id]
339 342
340 343 class EngineService(object, service.Service):
341 344 """Adapt a IPython shell into a IEngine implementing Twisted Service."""
342 345
343 346 zi.implements(IEngineBase)
344 347 name = 'EngineService'
345 348
346 349 def __init__(self, shellClass=Interpreter, mpi=None):
347 350 """Create an EngineService.
348 351
349 352 shellClass: something that implements IInterpreter or core1
350 353 mpi: an mpi module that has rank and size attributes
351 354 """
352 355 self.shellClass = shellClass
353 356 self.shell = self.shellClass()
354 357 self.mpi = mpi
355 358 self.id = None
356 359 self.properties = get_engine(self.id).properties
357 360 if self.mpi is not None:
358 361 log.msg("MPI started with rank = %i and size = %i" %
359 362 (self.mpi.rank, self.mpi.size))
360 363 self.id = self.mpi.rank
361 364 self._seedNamespace()
362 365
363 366 # Make id a property so that the shell can get the updated id
364 367
365 368 def _setID(self, id):
366 369 self._id = id
367 370 self.properties = get_engine(id).properties
368 371 self.shell.push({'id': id})
369 372
370 373 def _getID(self):
371 374 return self._id
372 375
373 376 id = property(_getID, _setID)
374 377
375 378 def _seedNamespace(self):
376 379 self.shell.push({'mpi': self.mpi, 'id' : self.id})
377 380
378 381 def executeAndRaise(self, msg, callable, *args, **kwargs):
379 382 """Call a method of self.shell and wrap any exception."""
380 383 d = defer.Deferred()
381 384 try:
382 385 result = callable(*args, **kwargs)
383 386 except:
384 387 # This gives the following:
385 388 # et=exception class
386 389 # ev=exception class instance
387 390 # tb=traceback object
388 391 et,ev,tb = sys.exc_info()
389 392 # This call adds attributes to the exception value
390 393 et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
391 394 # Add another attribute
392 395 ev._ipython_engine_info = msg
393 396 f = failure.Failure(ev,et,None)
394 397 d.errback(f)
395 398 else:
396 399 d.callback(result)
397 400
398 401 return d
399 402
400 403
401 404 # The IEngine methods. See the interface for documentation.
402 405
403 406 def execute(self, lines):
404 407 msg = {'engineid':self.id,
405 408 'method':'execute',
406 409 'args':[lines]}
407 410 d = self.executeAndRaise(msg, self.shell.execute, lines)
408 411 d.addCallback(self.addIDToResult)
409 412 return d
410 413
411 414 def addIDToResult(self, result):
412 415 result['id'] = self.id
413 416 return result
414 417
415 418 def push(self, namespace):
416 419 msg = {'engineid':self.id,
417 420 'method':'push',
418 421 'args':[repr(namespace.keys())]}
419 422 d = self.executeAndRaise(msg, self.shell.push, namespace)
420 423 return d
421 424
422 425 def pull(self, keys):
423 426 msg = {'engineid':self.id,
424 427 'method':'pull',
425 428 'args':[repr(keys)]}
426 429 d = self.executeAndRaise(msg, self.shell.pull, keys)
427 430 return d
428 431
429 432 def push_function(self, namespace):
430 433 msg = {'engineid':self.id,
431 434 'method':'push_function',
432 435 'args':[repr(namespace.keys())]}
433 436 d = self.executeAndRaise(msg, self.shell.push_function, namespace)
434 437 return d
435 438
436 439 def pull_function(self, keys):
437 440 msg = {'engineid':self.id,
438 441 'method':'pull_function',
439 442 'args':[repr(keys)]}
440 443 d = self.executeAndRaise(msg, self.shell.pull_function, keys)
441 444 return d
442 445
443 446 def get_result(self, i=None):
444 447 msg = {'engineid':self.id,
445 448 'method':'get_result',
446 449 'args':[repr(i)]}
447 450 d = self.executeAndRaise(msg, self.shell.getCommand, i)
448 451 d.addCallback(self.addIDToResult)
449 452 return d
450 453
451 454 def reset(self):
452 455 msg = {'engineid':self.id,
453 456 'method':'reset',
454 457 'args':[]}
455 458 del self.shell
456 459 self.shell = self.shellClass()
457 460 self.properties.clear()
458 461 d = self.executeAndRaise(msg, self._seedNamespace)
459 462 return d
460 463
461 464 def kill(self):
462 465 drop_engine(self.id)
463 466 try:
464 467 reactor.stop()
465 468 except RuntimeError:
466 469 log.msg('The reactor was not running apparently.')
467 470 return defer.fail()
468 471 else:
469 472 return defer.succeed(None)
470 473
471 474 def keys(self):
472 475 """Return a list of variables names in the users top level namespace.
473 476
474 477 This used to return a dict of all the keys/repr(values) in the
475 478 user's namespace. This was too much info for the ControllerService
476 479 to handle so it is now just a list of keys.
477 480 """
478 481
479 482 remotes = []
480 483 for k in self.shell.user_ns.iterkeys():
481 484 if k not in ['__name__', '_ih', '_oh', '__builtins__',
482 485 'In', 'Out', '_', '__', '___', '__IP', 'input', 'raw_input']:
483 486 remotes.append(k)
484 487 return defer.succeed(remotes)
485 488
486 489 def set_properties(self, properties):
487 490 msg = {'engineid':self.id,
488 491 'method':'set_properties',
489 492 'args':[repr(properties.keys())]}
490 493 return self.executeAndRaise(msg, self.properties.update, properties)
491 494
492 495 def get_properties(self, keys=None):
493 496 msg = {'engineid':self.id,
494 497 'method':'get_properties',
495 498 'args':[repr(keys)]}
496 499 if keys is None:
497 500 keys = self.properties.keys()
498 501 return self.executeAndRaise(msg, self.properties.subDict, *keys)
499 502
500 503 def _doDel(self, keys):
501 504 for key in keys:
502 505 del self.properties[key]
503 506
504 507 def del_properties(self, keys):
505 508 msg = {'engineid':self.id,
506 509 'method':'del_properties',
507 510 'args':[repr(keys)]}
508 511 return self.executeAndRaise(msg, self._doDel, keys)
509 512
510 513 def _doHas(self, keys):
511 514 return [self.properties.has_key(key) for key in keys]
512 515
513 516 def has_properties(self, keys):
514 517 msg = {'engineid':self.id,
515 518 'method':'has_properties',
516 519 'args':[repr(keys)]}
517 520 return self.executeAndRaise(msg, self._doHas, keys)
518 521
519 522 def clear_properties(self):
520 523 msg = {'engineid':self.id,
521 524 'method':'clear_properties',
522 525 'args':[]}
523 526 return self.executeAndRaise(msg, self.properties.clear)
524 527
525 528 def push_serialized(self, sNamespace):
526 529 msg = {'engineid':self.id,
527 530 'method':'push_serialized',
528 531 'args':[repr(sNamespace.keys())]}
529 532 ns = {}
530 533 for k,v in sNamespace.iteritems():
531 534 try:
532 535 unserialized = newserialized.IUnSerialized(v)
533 536 ns[k] = unserialized.getObject()
534 537 except:
535 538 return defer.fail()
536 539 return self.executeAndRaise(msg, self.shell.push, ns)
537 540
538 541 def pull_serialized(self, keys):
539 542 msg = {'engineid':self.id,
540 543 'method':'pull_serialized',
541 544 'args':[repr(keys)]}
542 545 if isinstance(keys, str):
543 546 keys = [keys]
544 547 if len(keys)==1:
545 548 d = self.executeAndRaise(msg, self.shell.pull, keys)
546 549 d.addCallback(newserialized.serialize)
547 550 return d
548 551 elif len(keys)>1:
549 552 d = self.executeAndRaise(msg, self.shell.pull, keys)
550 553 @d.addCallback
551 554 def packThemUp(values):
552 555 serials = []
553 556 for v in values:
554 557 try:
555 558 serials.append(newserialized.serialize(v))
556 559 except:
557 560 return defer.fail(failure.Failure())
558 561 return serials
559 562 return packThemUp
560 563
561 564
562 565 def queue(methodToQueue):
563 566 def queuedMethod(this, *args, **kwargs):
564 567 name = methodToQueue.__name__
565 568 return this.submitCommand(Command(name, *args, **kwargs))
566 569 return queuedMethod
567 570
568 571 class QueuedEngine(object):
569 572 """Adapt an IEngineBase to an IEngineQueued by wrapping it.
570 573
571 574 The resulting object will implement IEngineQueued which extends
572 575 IEngineCore which extends (IEngineBase, IEngineSerialized).
573 576
574 577 This seems like the best way of handling it, but I am not sure. The
575 578 other option is to have the various base interfaces be used like
576 579 mix-in intefaces. The problem I have with this is adpatation is
577 580 more difficult and complicated because there can be can multiple
578 581 original and final Interfaces.
579 582 """
580 583
581 584 zi.implements(IEngineQueued)
582 585
583 586 def __init__(self, engine):
584 587 """Create a QueuedEngine object from an engine
585 588
586 589 engine: An implementor of IEngineCore and IEngineSerialized
587 590 keepUpToDate: whether to update the remote status when the
588 591 queue is empty. Defaults to False.
589 592 """
590 593
591 594 # This is the right way to do these tests rather than
592 595 # IEngineCore in list(zi.providedBy(engine)) which will only
593 596 # picks of the interfaces that are directly declared by engine.
594 597 assert IEngineBase.providedBy(engine), \
595 598 "engine passed to QueuedEngine doesn't provide IEngineBase"
596 599
597 600 self.engine = engine
598 601 self.id = engine.id
599 602 self.queued = []
600 603 self.history = {}
601 604 self.engineStatus = {}
602 605 self.currentCommand = None
603 606 self.failureObservers = []
604 607
605 608 def _get_properties(self):
606 609 return self.engine.properties
607 610
608 611 properties = property(_get_properties, lambda self, _: None)
609 612 # Queue management methods. You should not call these directly
610 613
611 614 def submitCommand(self, cmd):
612 615 """Submit command to queue."""
613 616
614 617 d = defer.Deferred()
615 618 cmd.setDeferred(d)
616 619 if self.currentCommand is not None:
617 620 if self.currentCommand.finished:
618 621 # log.msg("Running command immediately: %r" % cmd)
619 622 self.currentCommand = cmd
620 623 self.runCurrentCommand()
621 624 else: # command is still running
622 625 # log.msg("Command is running: %r" % self.currentCommand)
623 626 # log.msg("Queueing: %r" % cmd)
624 627 self.queued.append(cmd)
625 628 else:
626 629 # log.msg("No current commands, running: %r" % cmd)
627 630 self.currentCommand = cmd
628 631 self.runCurrentCommand()
629 632 return d
630 633
631 634 def runCurrentCommand(self):
632 635 """Run current command."""
633 636
634 637 cmd = self.currentCommand
635 638 f = getattr(self.engine, cmd.remoteMethod, None)
636 639 if f:
637 640 d = f(*cmd.args, **cmd.kwargs)
638 641 if cmd.remoteMethod is 'execute':
639 642 d.addCallback(self.saveResult)
640 643 d.addCallback(self.finishCommand)
641 644 d.addErrback(self.abortCommand)
642 645 else:
643 646 return defer.fail(AttributeError(cmd.remoteMethod))
644 647
645 648 def _flushQueue(self):
646 649 """Pop next command in queue and run it."""
647 650
648 651 if len(self.queued) > 0:
649 652 self.currentCommand = self.queued.pop(0)
650 653 self.runCurrentCommand()
651 654
652 655 def saveResult(self, result):
653 656 """Put the result in the history."""
654 657 self.history[result['number']] = result
655 658 return result
656 659
657 660 def finishCommand(self, result):
658 661 """Finish currrent command."""
659 662
660 663 # The order of these commands is absolutely critical.
661 664 self.currentCommand.handleResult(result)
662 665 self.currentCommand.finished = True
663 666 self._flushQueue()
664 667 return result
665 668
666 669 def abortCommand(self, reason):
667 670 """Abort current command.
668 671
669 672 This eats the Failure but first passes it onto the Deferred that the
670 673 user has.
671 674
672 675 It also clear out the queue so subsequence commands don't run.
673 676 """
674 677
675 678 # The order of these 3 commands is absolutely critical. The currentCommand
676 679 # must first be marked as finished BEFORE the queue is cleared and before
677 680 # the current command is sent the failure.
678 681 # Also, the queue must be cleared BEFORE the current command is sent the Failure
679 682 # otherwise the errback chain could trigger new commands to be added to the
680 683 # queue before we clear it. We should clear ONLY the commands that were in
681 684 # the queue when the error occured.
682 685 self.currentCommand.finished = True
683 686 s = "%r %r %r" % (self.currentCommand.remoteMethod, self.currentCommand.args, self.currentCommand.kwargs)
684 687 self.clear_queue(msg=s)
685 688 self.currentCommand.handleError(reason)
686 689
687 690 return None
688 691
689 692 #---------------------------------------------------------------------------
690 693 # IEngineCore methods
691 694 #---------------------------------------------------------------------------
692 695
693 696 @queue
694 697 def execute(self, lines):
695 698 pass
696 699
697 700 @queue
698 701 def push(self, namespace):
699 702 pass
700 703
701 704 @queue
702 705 def pull(self, keys):
703 706 pass
704 707
705 708 @queue
706 709 def push_function(self, namespace):
707 710 pass
708 711
709 712 @queue
710 713 def pull_function(self, keys):
711 714 pass
712 715
713 716 def get_result(self, i=None):
714 717 if i is None:
715 718 i = max(self.history.keys()+[None])
716 719
717 720 cmd = self.history.get(i, None)
718 721 # Uncomment this line to disable chaching of results
719 722 #cmd = None
720 723 if cmd is None:
721 724 return self.submitCommand(Command('get_result', i))
722 725 else:
723 726 return defer.succeed(cmd)
724 727
725 728 def reset(self):
726 729 self.clear_queue()
727 730 self.history = {} # reset the cache - I am not sure we should do this
728 731 return self.submitCommand(Command('reset'))
729 732
730 733 def kill(self):
731 734 self.clear_queue()
732 735 return self.submitCommand(Command('kill'))
733 736
734 737 @queue
735 738 def keys(self):
736 739 pass
737 740
738 741 #---------------------------------------------------------------------------
739 742 # IEngineSerialized methods
740 743 #---------------------------------------------------------------------------
741 744
742 745 @queue
743 746 def push_serialized(self, namespace):
744 747 pass
745 748
746 749 @queue
747 750 def pull_serialized(self, keys):
748 751 pass
749 752
750 753 #---------------------------------------------------------------------------
751 754 # IEngineProperties methods
752 755 #---------------------------------------------------------------------------
753 756
754 757 @queue
755 758 def set_properties(self, namespace):
756 759 pass
757 760
758 761 @queue
759 762 def get_properties(self, keys=None):
760 763 pass
761 764
762 765 @queue
763 766 def del_properties(self, keys):
764 767 pass
765 768
766 769 @queue
767 770 def has_properties(self, keys):
768 771 pass
769 772
770 773 @queue
771 774 def clear_properties(self):
772 775 pass
773 776
774 777 #---------------------------------------------------------------------------
775 778 # IQueuedEngine methods
776 779 #---------------------------------------------------------------------------
777 780
778 781 def clear_queue(self, msg=''):
779 782 """Clear the queue, but doesn't cancel the currently running commmand."""
780 783
781 784 for cmd in self.queued:
782 785 cmd.deferred.errback(failure.Failure(error.QueueCleared(msg)))
783 786 self.queued = []
784 787 return defer.succeed(None)
785 788
786 789 def queue_status(self):
787 790 if self.currentCommand is not None:
788 791 if self.currentCommand.finished:
789 792 pending = repr(None)
790 793 else:
791 794 pending = repr(self.currentCommand)
792 795 else:
793 796 pending = repr(None)
794 797 dikt = {'queue':map(repr,self.queued), 'pending':pending}
795 798 return defer.succeed(dikt)
796 799
797 800 def register_failure_observer(self, obs):
798 801 self.failureObservers.append(obs)
799 802
800 803 def unregister_failure_observer(self, obs):
801 804 self.failureObservers.remove(obs)
802 805
803 806
804 807 # Now register QueuedEngine as an adpater class that makes an IEngineBase into a
805 808 # IEngineQueued.
806 809 components.registerAdapter(QueuedEngine, IEngineBase, IEngineQueued)
807 810
808 811
809 812 class Command(object):
810 813 """A command object that encapslates queued commands.
811 814
812 815 This class basically keeps track of a command that has been queued
813 816 in a QueuedEngine. It manages the deferreds and hold the method to be called
814 817 and the arguments to that method.
815 818 """
816 819
817 820
818 821 def __init__(self, remoteMethod, *args, **kwargs):
819 822 """Build a new Command object."""
820 823
821 824 self.remoteMethod = remoteMethod
822 825 self.args = args
823 826 self.kwargs = kwargs
824 827 self.finished = False
825 828
826 829 def setDeferred(self, d):
827 830 """Sets the deferred attribute of the Command."""
828 831
829 832 self.deferred = d
830 833
831 834 def __repr__(self):
832 835 if not self.args:
833 836 args = ''
834 837 else:
835 838 args = str(self.args)[1:-2] #cut off (...,)
836 839 for k,v in self.kwargs.iteritems():
837 840 if args:
838 841 args += ', '
839 842 args += '%s=%r' %(k,v)
840 843 return "%s(%s)" %(self.remoteMethod, args)
841 844
842 845 def handleResult(self, result):
843 846 """When the result is ready, relay it to self.deferred."""
844 847
845 848 self.deferred.callback(result)
846 849
847 850 def handleError(self, reason):
848 851 """When an error has occured, relay it to self.deferred."""
849 852
850 853 self.deferred.errback(reason)
851 854
852 855 class ThreadedEngineService(EngineService):
853 856 """An EngineService subclass that defers execute commands to a separate
854 857 thread.
855 858
856 859 ThreadedEngineService uses twisted.internet.threads.deferToThread to
857 860 defer execute requests to a separate thread. GUI frontends may want to
858 861 use ThreadedEngineService as the engine in an
859 862 IPython.frontend.frontendbase.FrontEndBase subclass to prevent
860 863 block execution from blocking the GUI thread.
861 864 """
862 865
863 866 zi.implements(IEngineBase)
864 867
865 868 def __init__(self, shellClass=Interpreter, mpi=None):
866 869 EngineService.__init__(self, shellClass, mpi)
867 870
868 871 def wrapped_execute(self, msg, lines):
869 872 """Wrap self.shell.execute to add extra information to tracebacks"""
870 873
871 874 try:
872 875 result = self.shell.execute(lines)
873 876 except Exception,e:
874 877 # This gives the following:
875 878 # et=exception class
876 879 # ev=exception class instance
877 880 # tb=traceback object
878 881 et,ev,tb = sys.exc_info()
879 882 # This call adds attributes to the exception value
880 883 et,ev,tb = self.shell.formatTraceback(et,ev,tb,msg)
881 884 # Add another attribute
882 885
883 886 # Create a new exception with the new attributes
884 887 e = et(ev._ipython_traceback_text)
885 888 e._ipython_engine_info = msg
886 889
887 890 # Re-raise
888 891 raise e
889 892
890 893 return result
891 894
892 895
893 896 def execute(self, lines):
894 897 # Only import this if we are going to use this class
895 898 from twisted.internet import threads
896 899
897 900 msg = {'engineid':self.id,
898 901 'method':'execute',
899 902 'args':[lines]}
900 903
901 904 d = threads.deferToThread(self.wrapped_execute, msg, lines)
902 905 d.addCallback(self.addIDToResult)
903 906 return d
@@ -1,1113 +1,1116 b''
1 1 # encoding: utf-8
2 2 # -*- test-case-name: IPython.kernel.tests.test_task -*-
3 3
4 4 """Task farming representation of the ControllerService."""
5 5
6 6 __docformat__ = "restructuredtext en"
7 7
8 8 #-----------------------------------------------------------------------------
9 9 # Copyright (C) 2008 The IPython Development Team
10 10 #
11 11 # Distributed under the terms of the BSD License. The full license is in
12 12 # the file COPYING, distributed as part of this software.
13 13 #-----------------------------------------------------------------------------
14 14
15 15 #-----------------------------------------------------------------------------
16 16 # Imports
17 17 #-----------------------------------------------------------------------------
18 18
19 # Tell nose to skip the testing of this module
20 __test__ = {}
21
19 22 import copy, time
20 23 from types import FunctionType
21 24
22 25 import zope.interface as zi, string
23 26 from twisted.internet import defer, reactor
24 27 from twisted.python import components, log, failure
25 28
26 29 from IPython.kernel.util import printer
27 30 from IPython.kernel import engineservice as es, error
28 31 from IPython.kernel import controllerservice as cs
29 32 from IPython.kernel.twistedutil import gatherBoth, DeferredList
30 33
31 34 from IPython.kernel.pickleutil import can, uncan, CannedFunction
32 35
33 36 #-----------------------------------------------------------------------------
34 37 # Definition of the Task objects
35 38 #-----------------------------------------------------------------------------
36 39
37 40 time_format = '%Y/%m/%d %H:%M:%S'
38 41
39 42 class ITask(zi.Interface):
40 43 """
41 44 This interface provides a generic definition of what constitutes a task.
42 45
43 46 There are two sides to a task. First a task needs to take input from
44 47 a user to determine what work is performed by the task. Second, the
45 48 task needs to have the logic that knows how to turn that information
46 49 info specific calls to a worker, through the `IQueuedEngine` interface.
47 50
48 51 Many method in this class get two things passed to them: a Deferred
49 52 and an IQueuedEngine implementer. Such methods should register callbacks
50 53 on the Deferred that use the IQueuedEngine to accomplish something. See
51 54 the existing task objects for examples.
52 55 """
53 56
54 57 zi.Attribute('retries','How many times to retry the task')
55 58 zi.Attribute('recovery_task','A task to try if the initial one fails')
56 59 zi.Attribute('taskid','the id of the task')
57 60
58 61 def start_time(result):
59 62 """
60 63 Do anything needed to start the timing of the task.
61 64
62 65 Must simply return the result after starting the timers.
63 66 """
64 67
65 68 def stop_time(result):
66 69 """
67 70 Do anything needed to stop the timing of the task.
68 71
69 72 Must simply return the result after stopping the timers. This
70 73 method will usually set attributes that are used by `process_result`
71 74 in building result of the task.
72 75 """
73 76
74 77 def pre_task(d, queued_engine):
75 78 """Do something with the queued_engine before the task is run.
76 79
77 80 This method should simply add callbacks to the input Deferred
78 81 that do something with the `queued_engine` before the task is run.
79 82
80 83 :Parameters:
81 84 d : Deferred
82 85 The deferred that actions should be attached to
83 86 queued_engine : IQueuedEngine implementer
84 87 The worker that has been allocated to perform the task
85 88 """
86 89
87 90 def post_task(d, queued_engine):
88 91 """Do something with the queued_engine after the task is run.
89 92
90 93 This method should simply add callbacks to the input Deferred
91 94 that do something with the `queued_engine` before the task is run.
92 95
93 96 :Parameters:
94 97 d : Deferred
95 98 The deferred that actions should be attached to
96 99 queued_engine : IQueuedEngine implementer
97 100 The worker that has been allocated to perform the task
98 101 """
99 102
100 103 def submit_task(d, queued_engine):
101 104 """Submit a task using the `queued_engine` we have been allocated.
102 105
103 106 When a task is ready to run, this method is called. This method
104 107 must take the internal information of the task and make suitable
105 108 calls on the queued_engine to have the actual work done.
106 109
107 110 This method should simply add callbacks to the input Deferred
108 111 that do something with the `queued_engine` before the task is run.
109 112
110 113 :Parameters:
111 114 d : Deferred
112 115 The deferred that actions should be attached to
113 116 queued_engine : IQueuedEngine implementer
114 117 The worker that has been allocated to perform the task
115 118 """
116 119
117 120 def process_result(d, result, engine_id):
118 121 """Take a raw task result.
119 122
120 123 Objects that implement `ITask` can choose how the result of running
121 124 the task is presented. This method takes the raw result and
122 125 does this logic. Two example are the `MapTask` which simply returns
123 126 the raw result or a `Failure` object and the `StringTask` which
124 127 returns a `TaskResult` object.
125 128
126 129 :Parameters:
127 130 d : Deferred
128 131 The deferred that actions should be attached to
129 132 result : object
130 133 The raw task result that needs to be wrapped
131 134 engine_id : int
132 135 The id of the engine that did the task
133 136
134 137 :Returns:
135 138 The result, as a tuple of the form: (success, result).
136 139 Here, success is a boolean indicating if the task
137 140 succeeded or failed and result is the result.
138 141 """
139 142
140 143 def check_depend(properties):
141 144 """Check properties to see if the task should be run.
142 145
143 146 :Parameters:
144 147 properties : dict
145 148 A dictionary of properties that an engine has set
146 149
147 150 :Returns:
148 151 True if the task should be run, False otherwise
149 152 """
150 153
151 154 def can_task(self):
152 155 """Serialize (can) any functions in the task for pickling.
153 156
154 157 Subclasses must override this method and make sure that all
155 158 functions in the task are canned by calling `can` on the
156 159 function.
157 160 """
158 161
159 162 def uncan_task(self):
160 163 """Unserialize (uncan) any canned function in the task."""
161 164
162 165 class BaseTask(object):
163 166 """
164 167 Common fuctionality for all objects implementing `ITask`.
165 168 """
166 169
167 170 zi.implements(ITask)
168 171
169 172 def __init__(self, clear_before=False, clear_after=False, retries=0,
170 173 recovery_task=None, depend=None):
171 174 """
172 175 Make a generic task.
173 176
174 177 :Parameters:
175 178 clear_before : boolean
176 179 Should the engines namespace be cleared before the task
177 180 is run
178 181 clear_after : boolean
179 182 Should the engines namespace be clear after the task is run
180 183 retries : int
181 184 The number of times a task should be retries upon failure
182 185 recovery_task : any task object
183 186 If a task fails and it has a recovery_task, that is run
184 187 upon a retry
185 188 depend : FunctionType
186 189 A function that is called to test for properties. This function
187 190 must take one argument, the properties dict and return a boolean
188 191 """
189 192 self.clear_before = clear_before
190 193 self.clear_after = clear_after
191 194 self.retries = retries
192 195 self.recovery_task = recovery_task
193 196 self.depend = depend
194 197 self.taskid = None
195 198
196 199 def start_time(self, result):
197 200 """
198 201 Start the basic timers.
199 202 """
200 203 self.start = time.time()
201 204 self.start_struct = time.localtime()
202 205 return result
203 206
204 207 def stop_time(self, result):
205 208 """
206 209 Stop the basic timers.
207 210 """
208 211 self.stop = time.time()
209 212 self.stop_struct = time.localtime()
210 213 self.duration = self.stop - self.start
211 214 self.submitted = time.strftime(time_format, self.start_struct)
212 215 self.completed = time.strftime(time_format)
213 216 return result
214 217
215 218 def pre_task(self, d, queued_engine):
216 219 """
217 220 Clear the engine before running the task if clear_before is set.
218 221 """
219 222 if self.clear_before:
220 223 d.addCallback(lambda r: queued_engine.reset())
221 224
222 225 def post_task(self, d, queued_engine):
223 226 """
224 227 Clear the engine after running the task if clear_after is set.
225 228 """
226 229 def reseter(result):
227 230 queued_engine.reset()
228 231 return result
229 232 if self.clear_after:
230 233 d.addBoth(reseter)
231 234
232 235 def submit_task(self, d, queued_engine):
233 236 raise NotImplementedError('submit_task must be implemented in a subclass')
234 237
235 238 def process_result(self, result, engine_id):
236 239 """
237 240 Process a task result.
238 241
239 242 This is the default `process_result` that just returns the raw
240 243 result or a `Failure`.
241 244 """
242 245 if isinstance(result, failure.Failure):
243 246 return (False, result)
244 247 else:
245 248 return (True, result)
246 249
247 250 def check_depend(self, properties):
248 251 """
249 252 Calls self.depend(properties) to see if a task should be run.
250 253 """
251 254 if self.depend is not None:
252 255 return self.depend(properties)
253 256 else:
254 257 return True
255 258
256 259 def can_task(self):
257 260 self.depend = can(self.depend)
258 261 if isinstance(self.recovery_task, BaseTask):
259 262 self.recovery_task.can_task()
260 263
261 264 def uncan_task(self):
262 265 self.depend = uncan(self.depend)
263 266 if isinstance(self.recovery_task, BaseTask):
264 267 self.recovery_task.uncan_task()
265 268
266 269 class MapTask(BaseTask):
267 270 """
268 271 A task that consists of a function and arguments.
269 272 """
270 273
271 274 zi.implements(ITask)
272 275
273 276 def __init__(self, function, args=None, kwargs=None, clear_before=False,
274 277 clear_after=False, retries=0, recovery_task=None, depend=None):
275 278 """
276 279 Create a task based on a function, args and kwargs.
277 280
278 281 This is a simple type of task that consists of calling:
279 282 function(*args, **kwargs) and wrapping the result in a `TaskResult`.
280 283
281 284 The return value of the function, or a `Failure` wrapping an
282 285 exception is the task result for this type of task.
283 286 """
284 287 BaseTask.__init__(self, clear_before, clear_after, retries,
285 288 recovery_task, depend)
286 289 if not isinstance(function, FunctionType):
287 290 raise TypeError('a task function must be a FunctionType')
288 291 self.function = function
289 292 if args is None:
290 293 self.args = ()
291 294 else:
292 295 self.args = args
293 296 if not isinstance(self.args, (list, tuple)):
294 297 raise TypeError('a task args must be a list or tuple')
295 298 if kwargs is None:
296 299 self.kwargs = {}
297 300 else:
298 301 self.kwargs = kwargs
299 302 if not isinstance(self.kwargs, dict):
300 303 raise TypeError('a task kwargs must be a dict')
301 304
302 305 def submit_task(self, d, queued_engine):
303 306 d.addCallback(lambda r: queued_engine.push_function(
304 307 dict(_ipython_task_function=self.function))
305 308 )
306 309 d.addCallback(lambda r: queued_engine.push(
307 310 dict(_ipython_task_args=self.args,_ipython_task_kwargs=self.kwargs))
308 311 )
309 312 d.addCallback(lambda r: queued_engine.execute(
310 313 '_ipython_task_result = _ipython_task_function(*_ipython_task_args,**_ipython_task_kwargs)')
311 314 )
312 315 d.addCallback(lambda r: queued_engine.pull('_ipython_task_result'))
313 316
314 317 def can_task(self):
315 318 self.function = can(self.function)
316 319 BaseTask.can_task(self)
317 320
318 321 def uncan_task(self):
319 322 self.function = uncan(self.function)
320 323 BaseTask.uncan_task(self)
321 324
322 325
323 326 class StringTask(BaseTask):
324 327 """
325 328 A task that consists of a string of Python code to run.
326 329 """
327 330
328 331 def __init__(self, expression, pull=None, push=None,
329 332 clear_before=False, clear_after=False, retries=0,
330 333 recovery_task=None, depend=None):
331 334 """
332 335 Create a task based on a Python expression and variables
333 336
334 337 This type of task lets you push a set of variables to the engines
335 338 namespace, run a Python string in that namespace and then bring back
336 339 a different set of Python variables as the result.
337 340
338 341 Because this type of task can return many results (through the
339 342 `pull` keyword argument) it returns a special `TaskResult` object
340 343 that wraps the pulled variables, statistics about the run and
341 344 any exceptions raised.
342 345 """
343 346 if not isinstance(expression, str):
344 347 raise TypeError('a task expression must be a string')
345 348 self.expression = expression
346 349
347 350 if pull==None:
348 351 self.pull = ()
349 352 elif isinstance(pull, str):
350 353 self.pull = (pull,)
351 354 elif isinstance(pull, (list, tuple)):
352 355 self.pull = pull
353 356 else:
354 357 raise TypeError('pull must be str or a sequence of strs')
355 358
356 359 if push==None:
357 360 self.push = {}
358 361 elif isinstance(push, dict):
359 362 self.push = push
360 363 else:
361 364 raise TypeError('push must be a dict')
362 365
363 366 BaseTask.__init__(self, clear_before, clear_after, retries,
364 367 recovery_task, depend)
365 368
366 369 def submit_task(self, d, queued_engine):
367 370 if self.push is not None:
368 371 d.addCallback(lambda r: queued_engine.push(self.push))
369 372
370 373 d.addCallback(lambda r: queued_engine.execute(self.expression))
371 374
372 375 if self.pull is not None:
373 376 d.addCallback(lambda r: queued_engine.pull(self.pull))
374 377 else:
375 378 d.addCallback(lambda r: None)
376 379
377 380 def process_result(self, result, engine_id):
378 381 if isinstance(result, failure.Failure):
379 382 tr = TaskResult(result, engine_id)
380 383 else:
381 384 if self.pull is None:
382 385 resultDict = {}
383 386 elif len(self.pull) == 1:
384 387 resultDict = {self.pull[0]:result}
385 388 else:
386 389 resultDict = dict(zip(self.pull, result))
387 390 tr = TaskResult(resultDict, engine_id)
388 391 # Assign task attributes
389 392 tr.submitted = self.submitted
390 393 tr.completed = self.completed
391 394 tr.duration = self.duration
392 395 if hasattr(self,'taskid'):
393 396 tr.taskid = self.taskid
394 397 else:
395 398 tr.taskid = None
396 399 if isinstance(result, failure.Failure):
397 400 return (False, tr)
398 401 else:
399 402 return (True, tr)
400 403
401 404 class ResultNS(object):
402 405 """
403 406 A dict like object for holding the results of a task.
404 407
405 408 The result namespace object for use in `TaskResult` objects as tr.ns.
406 409 It builds an object from a dictionary, such that it has attributes
407 410 according to the key,value pairs of the dictionary.
408 411
409 412 This works by calling setattr on ALL key,value pairs in the dict. If a user
410 413 chooses to overwrite the `__repr__` or `__getattr__` attributes, they can.
411 414 This can be a bad idea, as it may corrupt standard behavior of the
412 415 ns object.
413 416
414 417 Example
415 418 --------
416 419
417 420 >>> ns = ResultNS({'a':17,'foo':range(3)})
418 421 >>> print ns
419 422 NS{'a': 17, 'foo': [0, 1, 2]}
420 423 >>> ns.a
421 424 17
422 425 >>> ns['foo']
423 426 [0, 1, 2]
424 427 """
425 428 def __init__(self, dikt):
426 429 for k,v in dikt.iteritems():
427 430 setattr(self,k,v)
428 431
429 432 def __repr__(self):
430 433 l = dir(self)
431 434 d = {}
432 435 for k in l:
433 436 # do not print private objects
434 437 if k[:2] != '__' and k[-2:] != '__':
435 438 d[k] = getattr(self, k)
436 439 return "NS"+repr(d)
437 440
438 441 def __getitem__(self, key):
439 442 return getattr(self, key)
440 443
441 444 class TaskResult(object):
442 445 """
443 446 An object for returning task results for certain types of tasks.
444 447
445 448 This object encapsulates the results of a task. On task
446 449 success it will have a keys attribute that will have a list
447 450 of the variables that have been pulled back. These variables
448 451 are accessible as attributes of this class as well. On
449 452 success the failure attribute will be None.
450 453
451 454 In task failure, keys will be empty, but failure will contain
452 455 the failure object that encapsulates the remote exception.
453 456 One can also simply call the `raise_exception` method of
454 457 this class to re-raise any remote exception in the local
455 458 session.
456 459
457 460 The `TaskResult` has a `.ns` member, which is a property for access
458 461 to the results. If the Task had pull=['a', 'b'], then the
459 462 Task Result will have attributes `tr.ns.a`, `tr.ns.b` for those values.
460 463 Accessing `tr.ns` will raise the remote failure if the task failed.
461 464
462 465 The `engineid` attribute should have the `engineid` of the engine
463 466 that ran the task. But, because engines can come and go,
464 467 the `engineid` may not continue to be
465 468 valid or accurate.
466 469
467 470 The `taskid` attribute simply gives the `taskid` that the task
468 471 is tracked under.
469 472 """
470 473 taskid = None
471 474
472 475 def _getNS(self):
473 476 if isinstance(self.failure, failure.Failure):
474 477 return self.failure.raiseException()
475 478 else:
476 479 return self._ns
477 480
478 481 def _setNS(self, v):
479 482 raise Exception("the ns attribute cannot be changed")
480 483
481 484 ns = property(_getNS, _setNS)
482 485
483 486 def __init__(self, results, engineid):
484 487 self.engineid = engineid
485 488 if isinstance(results, failure.Failure):
486 489 self.failure = results
487 490 self.results = {}
488 491 else:
489 492 self.results = results
490 493 self.failure = None
491 494
492 495 self._ns = ResultNS(self.results)
493 496
494 497 self.keys = self.results.keys()
495 498
496 499 def __repr__(self):
497 500 if self.failure is not None:
498 501 contents = self.failure
499 502 else:
500 503 contents = self.results
501 504 return "TaskResult[ID:%r]:%r"%(self.taskid, contents)
502 505
503 506 def __getitem__(self, key):
504 507 if self.failure is not None:
505 508 self.raise_exception()
506 509 return self.results[key]
507 510
508 511 def raise_exception(self):
509 512 """Re-raise any remote exceptions in the local python session."""
510 513 if self.failure is not None:
511 514 self.failure.raiseException()
512 515
513 516
514 517 #-----------------------------------------------------------------------------
515 518 # The controller side of things
516 519 #-----------------------------------------------------------------------------
517 520
518 521 class IWorker(zi.Interface):
519 522 """The Basic Worker Interface.
520 523
521 524 A worked is a representation of an Engine that is ready to run tasks.
522 525 """
523 526
524 527 zi.Attribute("workerid", "the id of the worker")
525 528
526 529 def run(task):
527 530 """Run task in worker's namespace.
528 531
529 532 :Parameters:
530 533 task : a `Task` object
531 534
532 535 :Returns: `Deferred` to a tuple of (success, result) where
533 536 success if a boolean that signifies success or failure
534 537 and result is the task result.
535 538 """
536 539
537 540
538 541 class WorkerFromQueuedEngine(object):
539 542 """Adapt an `IQueuedEngine` to an `IWorker` object"""
540 543
541 544 zi.implements(IWorker)
542 545
543 546 def __init__(self, qe):
544 547 self.queuedEngine = qe
545 548 self.workerid = None
546 549
547 550 def _get_properties(self):
548 551 return self.queuedEngine.properties
549 552
550 553 properties = property(_get_properties, lambda self, _:None)
551 554
552 555 def run(self, task):
553 556 """Run task in worker's namespace.
554 557
555 558 This takes a task and calls methods on the task that actually
556 559 cause `self.queuedEngine` to do the task. See the methods of
557 560 `ITask` for more information about how these methods are called.
558 561
559 562 :Parameters:
560 563 task : a `Task` object
561 564
562 565 :Returns: `Deferred` to a tuple of (success, result) where
563 566 success if a boolean that signifies success or failure
564 567 and result is the task result.
565 568 """
566 569 d = defer.succeed(None)
567 570 d.addCallback(task.start_time)
568 571 task.pre_task(d, self.queuedEngine)
569 572 task.submit_task(d, self.queuedEngine)
570 573 task.post_task(d, self.queuedEngine)
571 574 d.addBoth(task.stop_time)
572 575 d.addBoth(task.process_result, self.queuedEngine.id)
573 576 # At this point, there will be (success, result) coming down the line
574 577 return d
575 578
576 579
577 580 components.registerAdapter(WorkerFromQueuedEngine, es.IEngineQueued, IWorker)
578 581
579 582 class IScheduler(zi.Interface):
580 583 """The interface for a Scheduler.
581 584 """
582 585 zi.Attribute("nworkers", "the number of unassigned workers")
583 586 zi.Attribute("ntasks", "the number of unscheduled tasks")
584 587 zi.Attribute("workerids", "a list of the worker ids")
585 588 zi.Attribute("taskids", "a list of the task ids")
586 589
587 590 def add_task(task, **flags):
588 591 """Add a task to the queue of the Scheduler.
589 592
590 593 :Parameters:
591 594 task : an `ITask` implementer
592 595 The task to be queued.
593 596 flags : dict
594 597 General keywords for more sophisticated scheduling
595 598 """
596 599
597 600 def pop_task(id=None):
598 601 """Pops a task object from the queue.
599 602
600 603 This gets the next task to be run. If no `id` is requested, the highest priority
601 604 task is returned.
602 605
603 606 :Parameters:
604 607 id
605 608 The id of the task to be popped. The default (None) is to return
606 609 the highest priority task.
607 610
608 611 :Returns: an `ITask` implementer
609 612
610 613 :Exceptions:
611 614 IndexError : raised if no taskid in queue
612 615 """
613 616
614 617 def add_worker(worker, **flags):
615 618 """Add a worker to the worker queue.
616 619
617 620 :Parameters:
618 621 worker : an `IWorker` implementer
619 622 flags : dict
620 623 General keywords for more sophisticated scheduling
621 624 """
622 625
623 626 def pop_worker(id=None):
624 627 """Pops an IWorker object that is ready to do work.
625 628
626 629 This gets the next IWorker that is ready to do work.
627 630
628 631 :Parameters:
629 632 id : if specified, will pop worker with workerid=id, else pops
630 633 highest priority worker. Defaults to None.
631 634
632 635 :Returns:
633 636 an IWorker object
634 637
635 638 :Exceptions:
636 639 IndexError : raised if no workerid in queue
637 640 """
638 641
639 642 def ready():
640 643 """Returns True if there is something to do, False otherwise"""
641 644
642 645 def schedule():
643 646 """Returns (worker,task) pair for the next task to be run."""
644 647
645 648
646 649 class FIFOScheduler(object):
647 650 """
648 651 A basic First-In-First-Out (Queue) Scheduler.
649 652
650 653 This is the default Scheduler for the `TaskController`.
651 654 See the docstrings for `IScheduler` for interface details.
652 655 """
653 656
654 657 zi.implements(IScheduler)
655 658
656 659 def __init__(self):
657 660 self.tasks = []
658 661 self.workers = []
659 662
660 663 def _ntasks(self):
661 664 return len(self.tasks)
662 665
663 666 def _nworkers(self):
664 667 return len(self.workers)
665 668
666 669 ntasks = property(_ntasks, lambda self, _:None)
667 670 nworkers = property(_nworkers, lambda self, _:None)
668 671
669 672 def _taskids(self):
670 673 return [t.taskid for t in self.tasks]
671 674
672 675 def _workerids(self):
673 676 return [w.workerid for w in self.workers]
674 677
675 678 taskids = property(_taskids, lambda self,_:None)
676 679 workerids = property(_workerids, lambda self,_:None)
677 680
678 681 def add_task(self, task, **flags):
679 682 self.tasks.append(task)
680 683
681 684 def pop_task(self, id=None):
682 685 if id is None:
683 686 return self.tasks.pop(0)
684 687 else:
685 688 for i in range(len(self.tasks)):
686 689 taskid = self.tasks[i].taskid
687 690 if id == taskid:
688 691 return self.tasks.pop(i)
689 692 raise IndexError("No task #%i"%id)
690 693
691 694 def add_worker(self, worker, **flags):
692 695 self.workers.append(worker)
693 696
694 697 def pop_worker(self, id=None):
695 698 if id is None:
696 699 return self.workers.pop(0)
697 700 else:
698 701 for i in range(len(self.workers)):
699 702 workerid = self.workers[i].workerid
700 703 if id == workerid:
701 704 return self.workers.pop(i)
702 705 raise IndexError("No worker #%i"%id)
703 706
704 707 def schedule(self):
705 708 for t in self.tasks:
706 709 for w in self.workers:
707 710 try:# do not allow exceptions to break this
708 711 # Allow the task to check itself using its
709 712 # check_depend method.
710 713 cando = t.check_depend(w.properties)
711 714 except:
712 715 cando = False
713 716 if cando:
714 717 return self.pop_worker(w.workerid), self.pop_task(t.taskid)
715 718 return None, None
716 719
717 720
718 721
719 722 class LIFOScheduler(FIFOScheduler):
720 723 """
721 724 A Last-In-First-Out (Stack) Scheduler.
722 725
723 726 This scheduler should naively reward fast engines by giving
724 727 them more jobs. This risks starvation, but only in cases with
725 728 low load, where starvation does not really matter.
726 729 """
727 730
728 731 def add_task(self, task, **flags):
729 732 # self.tasks.reverse()
730 733 self.tasks.insert(0, task)
731 734 # self.tasks.reverse()
732 735
733 736 def add_worker(self, worker, **flags):
734 737 # self.workers.reverse()
735 738 self.workers.insert(0, worker)
736 739 # self.workers.reverse()
737 740
738 741
739 742 class ITaskController(cs.IControllerBase):
740 743 """
741 744 The Task based interface to a `ControllerService` object
742 745
743 746 This adapts a `ControllerService` to the ITaskController interface.
744 747 """
745 748
746 749 def run(task):
747 750 """
748 751 Run a task.
749 752
750 753 :Parameters:
751 754 task : an IPython `Task` object
752 755
753 756 :Returns: the integer ID of the task
754 757 """
755 758
756 759 def get_task_result(taskid, block=False):
757 760 """
758 761 Get the result of a task by its ID.
759 762
760 763 :Parameters:
761 764 taskid : int
762 765 the id of the task whose result is requested
763 766
764 767 :Returns: `Deferred` to the task result if the task is done, and None
765 768 if not.
766 769
767 770 :Exceptions:
768 771 actualResult will be an `IndexError` if no such task has been submitted
769 772 """
770 773
771 774 def abort(taskid):
772 775 """Remove task from queue if task is has not been submitted.
773 776
774 777 If the task has already been submitted, wait for it to finish and discard
775 778 results and prevent resubmission.
776 779
777 780 :Parameters:
778 781 taskid : the id of the task to be aborted
779 782
780 783 :Returns:
781 784 `Deferred` to abort attempt completion. Will be None on success.
782 785
783 786 :Exceptions:
784 787 deferred will fail with `IndexError` if no such task has been submitted
785 788 or the task has already completed.
786 789 """
787 790
788 791 def barrier(taskids):
789 792 """
790 793 Block until the list of taskids are completed.
791 794
792 795 Returns None on success.
793 796 """
794 797
795 798 def spin():
796 799 """
797 800 Touch the scheduler, to resume scheduling without submitting a task.
798 801 """
799 802
800 803 def queue_status(verbose=False):
801 804 """
802 805 Get a dictionary with the current state of the task queue.
803 806
804 807 If verbose is True, then return lists of taskids, otherwise,
805 808 return the number of tasks with each status.
806 809 """
807 810
808 811 def clear():
809 812 """
810 813 Clear all previously run tasks from the task controller.
811 814
812 815 This is needed because the task controller keep all task results
813 816 in memory. This can be a problem is there are many completed
814 817 tasks. Users should call this periodically to clean out these
815 818 cached task results.
816 819 """
817 820
818 821
819 822 class TaskController(cs.ControllerAdapterBase):
820 823 """The Task based interface to a Controller object.
821 824
822 825 If you want to use a different scheduler, just subclass this and set
823 826 the `SchedulerClass` member to the *class* of your chosen scheduler.
824 827 """
825 828
826 829 zi.implements(ITaskController)
827 830 SchedulerClass = FIFOScheduler
828 831
829 832 timeout = 30
830 833
831 834 def __init__(self, controller):
832 835 self.controller = controller
833 836 self.controller.on_register_engine_do(self.registerWorker, True)
834 837 self.controller.on_unregister_engine_do(self.unregisterWorker, True)
835 838 self.taskid = 0
836 839 self.failurePenalty = 1 # the time in seconds to penalize
837 840 # a worker for failing a task
838 841 self.pendingTasks = {} # dict of {workerid:(taskid, task)}
839 842 self.deferredResults = {} # dict of {taskid:deferred}
840 843 self.finishedResults = {} # dict of {taskid:actualResult}
841 844 self.workers = {} # dict of {workerid:worker}
842 845 self.abortPending = [] # dict of {taskid:abortDeferred}
843 846 self.idleLater = None # delayed call object for timeout
844 847 self.scheduler = self.SchedulerClass()
845 848
846 849 for id in self.controller.engines.keys():
847 850 self.workers[id] = IWorker(self.controller.engines[id])
848 851 self.workers[id].workerid = id
849 852 self.schedule.add_worker(self.workers[id])
850 853
851 854 def registerWorker(self, id):
852 855 """Called by controller.register_engine."""
853 856 if self.workers.get(id):
854 857 raise ValueError("worker with id %s already exists. This should not happen." % id)
855 858 self.workers[id] = IWorker(self.controller.engines[id])
856 859 self.workers[id].workerid = id
857 860 if not self.pendingTasks.has_key(id):# if not working
858 861 self.scheduler.add_worker(self.workers[id])
859 862 self.distributeTasks()
860 863
861 864 def unregisterWorker(self, id):
862 865 """Called by controller.unregister_engine"""
863 866
864 867 if self.workers.has_key(id):
865 868 try:
866 869 self.scheduler.pop_worker(id)
867 870 except IndexError:
868 871 pass
869 872 self.workers.pop(id)
870 873
871 874 def _pendingTaskIDs(self):
872 875 return [t.taskid for t in self.pendingTasks.values()]
873 876
874 877 #---------------------------------------------------------------------------
875 878 # Interface methods
876 879 #---------------------------------------------------------------------------
877 880
878 881 def run(self, task):
879 882 """
880 883 Run a task and return `Deferred` to its taskid.
881 884 """
882 885 task.taskid = self.taskid
883 886 task.start = time.localtime()
884 887 self.taskid += 1
885 888 d = defer.Deferred()
886 889 self.scheduler.add_task(task)
887 890 log.msg('Queuing task: %i' % task.taskid)
888 891
889 892 self.deferredResults[task.taskid] = []
890 893 self.distributeTasks()
891 894 return defer.succeed(task.taskid)
892 895
893 896 def get_task_result(self, taskid, block=False):
894 897 """
895 898 Returns a `Deferred` to the task result, or None.
896 899 """
897 900 log.msg("Getting task result: %i" % taskid)
898 901 if self.finishedResults.has_key(taskid):
899 902 tr = self.finishedResults[taskid]
900 903 return defer.succeed(tr)
901 904 elif self.deferredResults.has_key(taskid):
902 905 if block:
903 906 d = defer.Deferred()
904 907 self.deferredResults[taskid].append(d)
905 908 return d
906 909 else:
907 910 return defer.succeed(None)
908 911 else:
909 912 return defer.fail(IndexError("task ID not registered: %r" % taskid))
910 913
911 914 def abort(self, taskid):
912 915 """
913 916 Remove a task from the queue if it has not been run already.
914 917 """
915 918 if not isinstance(taskid, int):
916 919 return defer.fail(failure.Failure(TypeError("an integer task id expected: %r" % taskid)))
917 920 try:
918 921 self.scheduler.pop_task(taskid)
919 922 except IndexError, e:
920 923 if taskid in self.finishedResults.keys():
921 924 d = defer.fail(IndexError("Task Already Completed"))
922 925 elif taskid in self.abortPending:
923 926 d = defer.fail(IndexError("Task Already Aborted"))
924 927 elif taskid in self._pendingTaskIDs():# task is pending
925 928 self.abortPending.append(taskid)
926 929 d = defer.succeed(None)
927 930 else:
928 931 d = defer.fail(e)
929 932 else:
930 933 d = defer.execute(self._doAbort, taskid)
931 934
932 935 return d
933 936
934 937 def barrier(self, taskids):
935 938 dList = []
936 939 if isinstance(taskids, int):
937 940 taskids = [taskids]
938 941 for id in taskids:
939 942 d = self.get_task_result(id, block=True)
940 943 dList.append(d)
941 944 d = DeferredList(dList, consumeErrors=1)
942 945 d.addCallbacks(lambda r: None)
943 946 return d
944 947
945 948 def spin(self):
946 949 return defer.succeed(self.distributeTasks())
947 950
948 951 def queue_status(self, verbose=False):
949 952 pending = self._pendingTaskIDs()
950 953 failed = []
951 954 succeeded = []
952 955 for k,v in self.finishedResults.iteritems():
953 956 if not isinstance(v, failure.Failure):
954 957 if hasattr(v,'failure'):
955 958 if v.failure is None:
956 959 succeeded.append(k)
957 960 else:
958 961 failed.append(k)
959 962 scheduled = self.scheduler.taskids
960 963 if verbose:
961 964 result = dict(pending=pending, failed=failed,
962 965 succeeded=succeeded, scheduled=scheduled)
963 966 else:
964 967 result = dict(pending=len(pending),failed=len(failed),
965 968 succeeded=len(succeeded),scheduled=len(scheduled))
966 969 return defer.succeed(result)
967 970
968 971 #---------------------------------------------------------------------------
969 972 # Queue methods
970 973 #---------------------------------------------------------------------------
971 974
972 975 def _doAbort(self, taskid):
973 976 """
974 977 Helper function for aborting a pending task.
975 978 """
976 979 log.msg("Task aborted: %i" % taskid)
977 980 result = failure.Failure(error.TaskAborted())
978 981 self._finishTask(taskid, result)
979 982 if taskid in self.abortPending:
980 983 self.abortPending.remove(taskid)
981 984
982 985 def _finishTask(self, taskid, result):
983 986 dlist = self.deferredResults.pop(taskid)
984 987 # result.taskid = taskid # The TaskResult should save the taskid
985 988 self.finishedResults[taskid] = result
986 989 for d in dlist:
987 990 d.callback(result)
988 991
989 992 def distributeTasks(self):
990 993 """
991 994 Distribute tasks while self.scheduler has things to do.
992 995 """
993 996 log.msg("distributing Tasks")
994 997 worker, task = self.scheduler.schedule()
995 998 if not worker and not task:
996 999 if self.idleLater and self.idleLater.called:# we are inside failIdle
997 1000 self.idleLater = None
998 1001 else:
999 1002 self.checkIdle()
1000 1003 return False
1001 1004 # else something to do:
1002 1005 while worker and task:
1003 1006 # get worker and task
1004 1007 # add to pending
1005 1008 self.pendingTasks[worker.workerid] = task
1006 1009 # run/link callbacks
1007 1010 d = worker.run(task)
1008 1011 log.msg("Running task %i on worker %i" %(task.taskid, worker.workerid))
1009 1012 d.addBoth(self.taskCompleted, task.taskid, worker.workerid)
1010 1013 worker, task = self.scheduler.schedule()
1011 1014 # check for idle timeout:
1012 1015 self.checkIdle()
1013 1016 return True
1014 1017
1015 1018 def checkIdle(self):
1016 1019 if self.idleLater and not self.idleLater.called:
1017 1020 self.idleLater.cancel()
1018 1021 if self.scheduler.ntasks and self.workers and \
1019 1022 self.scheduler.nworkers == len(self.workers):
1020 1023 self.idleLater = reactor.callLater(self.timeout, self.failIdle)
1021 1024 else:
1022 1025 self.idleLater = None
1023 1026
1024 1027 def failIdle(self):
1025 1028 if not self.distributeTasks():
1026 1029 while self.scheduler.ntasks:
1027 1030 t = self.scheduler.pop_task()
1028 1031 msg = "task %i failed to execute due to unmet dependencies"%t.taskid
1029 1032 msg += " for %i seconds"%self.timeout
1030 1033 log.msg("Task aborted by timeout: %i" % t.taskid)
1031 1034 f = failure.Failure(error.TaskTimeout(msg))
1032 1035 self._finishTask(t.taskid, f)
1033 1036 self.idleLater = None
1034 1037
1035 1038
1036 1039 def taskCompleted(self, success_and_result, taskid, workerid):
1037 1040 """This is the err/callback for a completed task."""
1038 1041 success, result = success_and_result
1039 1042 try:
1040 1043 task = self.pendingTasks.pop(workerid)
1041 1044 except:
1042 1045 # this should not happen
1043 1046 log.msg("Tried to pop bad pending task %i from worker %i"%(taskid, workerid))
1044 1047 log.msg("Result: %r"%result)
1045 1048 log.msg("Pending tasks: %s"%self.pendingTasks)
1046 1049 return
1047 1050
1048 1051 # Check if aborted while pending
1049 1052 aborted = False
1050 1053 if taskid in self.abortPending:
1051 1054 self._doAbort(taskid)
1052 1055 aborted = True
1053 1056
1054 1057 if not aborted:
1055 1058 if not success:
1056 1059 log.msg("Task %i failed on worker %i"% (taskid, workerid))
1057 1060 if task.retries > 0: # resubmit
1058 1061 task.retries -= 1
1059 1062 self.scheduler.add_task(task)
1060 1063 s = "Resubmitting task %i, %i retries remaining" %(taskid, task.retries)
1061 1064 log.msg(s)
1062 1065 self.distributeTasks()
1063 1066 elif isinstance(task.recovery_task, BaseTask) and \
1064 1067 task.recovery_task.retries > -1:
1065 1068 # retries = -1 is to prevent infinite recovery_task loop
1066 1069 task.retries = -1
1067 1070 task.recovery_task.taskid = taskid
1068 1071 task = task.recovery_task
1069 1072 self.scheduler.add_task(task)
1070 1073 s = "Recovering task %i, %i retries remaining" %(taskid, task.retries)
1071 1074 log.msg(s)
1072 1075 self.distributeTasks()
1073 1076 else: # done trying
1074 1077 self._finishTask(taskid, result)
1075 1078 # wait a second before readmitting a worker that failed
1076 1079 # it may have died, and not yet been unregistered
1077 1080 reactor.callLater(self.failurePenalty, self.readmitWorker, workerid)
1078 1081 else: # we succeeded
1079 1082 log.msg("Task completed: %i"% taskid)
1080 1083 self._finishTask(taskid, result)
1081 1084 self.readmitWorker(workerid)
1082 1085 else: # we aborted the task
1083 1086 if not success:
1084 1087 reactor.callLater(self.failurePenalty, self.readmitWorker, workerid)
1085 1088 else:
1086 1089 self.readmitWorker(workerid)
1087 1090
1088 1091 def readmitWorker(self, workerid):
1089 1092 """
1090 1093 Readmit a worker to the scheduler.
1091 1094
1092 1095 This is outside `taskCompleted` because of the `failurePenalty` being
1093 1096 implemented through `reactor.callLater`.
1094 1097 """
1095 1098
1096 1099 if workerid in self.workers.keys() and workerid not in self.pendingTasks.keys():
1097 1100 self.scheduler.add_worker(self.workers[workerid])
1098 1101 self.distributeTasks()
1099 1102
1100 1103 def clear(self):
1101 1104 """
1102 1105 Clear all previously run tasks from the task controller.
1103 1106
1104 1107 This is needed because the task controller keep all task results
1105 1108 in memory. This can be a problem is there are many completed
1106 1109 tasks. Users should call this periodically to clean out these
1107 1110 cached task results.
1108 1111 """
1109 1112 self.finishedResults = {}
1110 1113 return defer.succeed(None)
1111 1114
1112 1115
1113 1116 components.registerAdapter(TaskController, cs.IControllerBase, ITaskController)
@@ -1,244 +1,246 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) or trial recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 For now, this script requires that both nose and twisted are installed. This
16 16 will change in the future.
17 17 """
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Module imports
21 21 #-----------------------------------------------------------------------------
22 22
23 23 import os
24 24 import os.path as path
25 25 import sys
26 26 import subprocess
27 27 import time
28 28 import warnings
29 29
30 30 import nose.plugins.builtin
31 31 from nose.core import TestProgram
32 32
33 33 from IPython.testing.plugin.ipdoctest import IPythonDoctest
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Globals and constants
37 37 #-----------------------------------------------------------------------------
38 38
39 39 # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
40 40 # testing problems. We should strive to minimize the number of skipped
41 41 # modules, since this means untested code. As the testing machinery
42 42 # solidifies, this list should eventually become empty.
43 43 EXCLUDE = ['IPython/external/',
44 44 'IPython/platutils_win32',
45 45 'IPython/frontend/cocoa',
46 46 'IPython/frontend/process/winprocess.py',
47 47 'IPython_doctest_plugin',
48 48 'IPython/Gnuplot',
49 49 'IPython/Extensions/ipy_',
50 50 'IPython/Extensions/clearcmd',
51 51 'IPython/Extensions/PhysicalQIn',
52 52 'IPython/Extensions/scitedirector',
53 53 'IPython/Extensions/numeric_formats',
54 54 'IPython/testing/attic',
55 55 ]
56 56
57 57 #-----------------------------------------------------------------------------
58 58 # Functions and classes
59 59 #-----------------------------------------------------------------------------
60 60
61 61 def run_iptest():
62 62 """Run the IPython test suite using nose.
63 63
64 64 This function is called when this script is **not** called with the form
65 65 `iptest all`. It simply calls nose with appropriate command line flags
66 66 and accepts all of the standard nose arguments.
67 67 """
68 68
69 69 warnings.filterwarnings('ignore',
70 70 'This will be removed soon. Use IPython.testing.util instead')
71 71
72 72 argv = sys.argv + [
73 73 # Loading ipdoctest causes problems with Twisted.
74 74 # I am removing this as a temporary fix to get the
75 75 # test suite back into working shape. Our nose
76 76 # plugin needs to be gone through with a fine
77 77 # toothed comb to find what is causing the problem.
78 78 '--with-ipdoctest',
79 79 '--ipdoctest-tests','--ipdoctest-extension=txt',
80 80 '--detailed-errors',
81 81
82 82 # We add --exe because of setuptools' imbecility (it
83 83 # blindly does chmod +x on ALL files). Nose does the
84 84 # right thing and it tries to avoid executables,
85 85 # setuptools unfortunately forces our hand here. This
86 86 # has been discussed on the distutils list and the
87 87 # setuptools devs refuse to fix this problem!
88 88 '--exe',
89 89 ]
90 90
91 91 # Detect if any tests were required by explicitly calling an IPython
92 92 # submodule or giving a specific path
93 93 has_tests = False
94 94 for arg in sys.argv:
95 95 if 'IPython' in arg or arg.endswith('.py') or \
96 96 (':' in arg and '.py' in arg):
97 97 has_tests = True
98 98 break
99 99
100 100 # If nothing was specifically requested, test full IPython
101 101 if not has_tests:
102 102 argv.append('IPython')
103 103
104 104 # Construct list of plugins, omitting the existing doctest plugin, which
105 105 # ours replaces (and extends).
106 106 plugins = [IPythonDoctest(EXCLUDE)]
107 107 for p in nose.plugins.builtin.plugins:
108 108 plug = p()
109 109 if plug.name == 'doctest':
110 110 continue
111 111
112 112 #print '*** adding plugin:',plug.name # dbg
113 113 plugins.append(plug)
114 114
115 115 TestProgram(argv=argv,plugins=plugins)
116 116
117 117
118 118 class IPTester(object):
119 119 """Call that calls iptest or trial in a subprocess.
120 120 """
121 121 def __init__(self,runner='iptest',params=None):
122 122 """ """
123 123 if runner == 'iptest':
124 124 self.runner = ['iptest','-v']
125 125 else:
126 126 self.runner = ['trial']
127 127 if params is None:
128 128 params = []
129 129 if isinstance(params,str):
130 130 params = [params]
131 131 self.params = params
132 132
133 133 # Assemble call
134 134 self.call_args = self.runner+self.params
135 135
136 136 def run(self):
137 137 """Run the stored commands"""
138 138 return subprocess.call(self.call_args)
139 139
140 140
141 141 def make_runners():
142 142 """Define the modules and packages that need to be tested.
143 143 """
144 144
145 145 # This omits additional top-level modules that should not be doctested.
146 146 # XXX: Shell.py is also ommited because of a bug in the skip_doctest
147 147 # decorator. See ticket https://bugs.launchpad.net/bugs/366209
148 148 top_mod = \
149 149 ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
150 150 'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
151 151 'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
152 152 'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
153 153 'iplib.py', 'ipmaker.py', 'ipstruct.py', 'irunner.py', 'Itpl.py',
154 154 'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
155 155 'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
156 156 'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
157 157 'shellglobals.py', 'strdispatch.py', 'twshell.py',
158 158 'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
159 159 # See note above for why this is skipped
160 160 # 'Shell.py',
161 161 'winconsole.py']
162 162
163 163 if os.name == 'posix':
164 164 top_mod.append('platutils_posix.py')
165 165 elif sys.platform == 'win32':
166 166 top_mod.append('platutils_win32.py')
167 167 else:
168 168 top_mod.append('platutils_dummy.py')
169 169
170 top_pack = ['config','Extensions','frontend','gui','kernel',
170 # These are tested by nose, so skip IPython.kernel
171 top_pack = ['config','Extensions','frontend','gui',
171 172 'testing','tests','tools','UserConfig']
172 173
173 174 modules = ['IPython.%s' % m[:-3] for m in top_mod ]
174 175 packages = ['IPython.%s' % m for m in top_pack ]
175 176
176 177 # Make runners
177 178 runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
178 179
180 # Test IPython.kernel using trial if twisted is installed
179 181 try:
180 182 import zope.interface
181 183 import twisted
182 184 import foolscap
183 185 except ImportError:
184 186 pass
185 187 else:
186 188 runners['trial'] = IPTester('trial',['IPython'])
187 189
188 190 for m in modules:
189 191 runners[m] = IPTester(params=m)
190 192
191 193 return runners
192 194
193 195
194 196 def run_iptestall():
195 197 """Run the entire IPython test suite by calling nose and trial.
196 198
197 199 This function constructs :class:`IPTester` instances for all IPython
198 200 modules and package and then runs each of them. This causes the modules
199 201 and packages of IPython to be tested each in their own subprocess using
200 202 nose or twisted.trial appropriately.
201 203 """
202 204 runners = make_runners()
203 205 # Run all test runners, tracking execution time
204 206 failed = {}
205 207 t_start = time.time()
206 208 for name,runner in runners.iteritems():
207 209 print '*'*77
208 210 print 'IPython test set:',name
209 211 res = runner.run()
210 212 if res:
211 213 failed[name] = res
212 214 t_end = time.time()
213 215 t_tests = t_end - t_start
214 216 nrunners = len(runners)
215 217 nfail = len(failed)
216 218 # summarize results
217 219 print
218 220 print '*'*77
219 221 print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
220 222 print
221 223 if not failed:
222 224 print 'OK'
223 225 else:
224 226 # If anything went wrong, point out what command to rerun manually to
225 227 # see the actual errors and individual summary
226 228 print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
227 229 for name in failed:
228 230 failed_runner = runners[name]
229 231 print '-'*40
230 232 print 'Runner failed:',name
231 233 print 'You may wish to rerun this one individually, with:'
232 234 print ' '.join(failed_runner.call_args)
233 235 print
234 236
235 237
236 238 def main():
237 239 if sys.argv[1] == 'all':
238 240 run_iptestall()
239 241 else:
240 242 run_iptest()
241 243
242 244
243 245 if __name__ == '__main__':
244 246 main() No newline at end of file
@@ -1,889 +1,908 b''
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by starting ipython with the
7 7 flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
8 8 interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import __builtin__
23 23 import commands
24 24 import doctest
25 25 import inspect
26 26 import logging
27 27 import os
28 28 import re
29 29 import sys
30 30 import traceback
31 31 import unittest
32 32
33 33 from inspect import getmodule
34 34 from StringIO import StringIO
35 35
36 36 # We are overriding the default doctest runner, so we need to import a few
37 37 # things from doctest directly
38 38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 39 _unittest_reportflags, DocTestRunner,
40 40 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 41 _exception_traceback,
42 42 linecache)
43 43
44 44 # Third-party modules
45 45 import nose.core
46 46
47 47 from nose.plugins import doctests, Plugin
48 48 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module globals and other constants
52 52
53 53 log = logging.getLogger(__name__)
54 54
55 55 ###########################################################################
56 56 # *** HACK ***
57 57 # We must start our own ipython object and heavily muck with it so that all the
58 58 # modifications IPython makes to system behavior don't send the doctest
59 59 # machinery into a fit. This code should be considered a gross hack, but it
60 60 # gets the job done.
61 61
62 62 def default_argv():
63 63 """Return a valid default argv for creating testing instances of ipython"""
64 64
65 65 # Get the install directory for the user configuration and tell ipython to
66 66 # use the default profile from there.
67 67 from IPython import UserConfig
68 68 ipcdir = os.path.dirname(UserConfig.__file__)
69 69 #ipconf = os.path.join(ipcdir,'ipy_user_conf.py')
70 70 ipconf = os.path.join(ipcdir,'ipythonrc')
71 71 #print 'conf:',ipconf # dbg
72 72
73 73 return ['--colors=NoColor','--noterm_title','-rcfile=%s' % ipconf]
74 74
75 75
76 76 # Hack to modify the %run command so we can sync the user's namespace with the
77 77 # test globals. Once we move over to a clean magic system, this will be done
78 78 # with much less ugliness.
79 79
80 80 class py_file_finder(object):
81 81 def __init__(self,test_filename):
82 82 self.test_filename = test_filename
83 83
84 84 def __call__(self,name):
85 85 from IPython.genutils import get_py_filename
86 86 try:
87 87 return get_py_filename(name)
88 88 except IOError:
89 89 test_dir = os.path.dirname(self.test_filename)
90 90 new_path = os.path.join(test_dir,name)
91 91 return get_py_filename(new_path)
92 92
93 93
94 94 def _run_ns_sync(self,arg_s,runner=None):
95 95 """Modified version of %run that syncs testing namespaces.
96 96
97 97 This is strictly needed for running doctests that call %run.
98 98 """
99 99
100 finder = py_file_finder(_run_ns_sync.test_filename)
100 # When tests call %run directly (not via doctest) these function attributes
101 # are not set
102 try:
103 fname = _run_ns_sync.test_filename
104 except AttributeError:
105 fname = arg_s
106
107 finder = py_file_finder(fname)
101 108 out = _ip.IP.magic_run_ori(arg_s,runner,finder)
102 _run_ns_sync.test_globs.update(_ip.user_ns)
109
110 # Simliarly, there is no test_globs when a test is NOT a doctest
111 if hasattr(_run_ns_sync,'test_globs'):
112 _run_ns_sync.test_globs.update(_ip.user_ns)
103 113 return out
104 114
105 115
106 116 class ipnsdict(dict):
107 117 """A special subclass of dict for use as an IPython namespace in doctests.
108 118
109 119 This subclass adds a simple checkpointing capability so that when testing
110 120 machinery clears it (we use it as the test execution context), it doesn't
111 121 get completely destroyed.
112 122 """
113 123
114 124 def __init__(self,*a):
115 125 dict.__init__(self,*a)
116 126 self._savedict = {}
117 127
118 128 def clear(self):
119 129 dict.clear(self)
120 130 self.update(self._savedict)
121 131
122 132 def _checkpoint(self):
123 133 self._savedict.clear()
124 134 self._savedict.update(self)
125 135
126 136 def update(self,other):
127 137 self._checkpoint()
128 138 dict.update(self,other)
139
129 140 # If '_' is in the namespace, python won't set it when executing code,
130 141 # and we have examples that test it. So we ensure that the namespace
131 142 # is always 'clean' of it before it's used for test code execution.
132 143 self.pop('_',None)
144
145 # The builtins namespace must *always* be the real __builtin__ module,
146 # else weird stuff happens. The main ipython code does have provisions
147 # to ensure this after %run, but since in this class we do some
148 # aggressive low-level cleaning of the execution namespace, we need to
149 # correct for that ourselves, to ensure consitency with the 'real'
150 # ipython.
151 self['__builtins__'] = __builtin__
133 152
134 153
135 154 def start_ipython():
136 155 """Start a global IPython shell, which we need for IPython-specific syntax.
137 156 """
138 157
139 158 # This function should only ever run once!
140 159 if hasattr(start_ipython,'already_called'):
141 160 return
142 161 start_ipython.already_called = True
143 162
144 163 # Ok, first time we're called, go ahead
145 164 import new
146 165
147 166 import IPython
148 167
149 168 def xsys(cmd):
150 169 """Execute a command and print its output.
151 170
152 171 This is just a convenience function to replace the IPython system call
153 172 with one that is more doctest-friendly.
154 173 """
155 174 cmd = _ip.IP.var_expand(cmd,depth=1)
156 175 sys.stdout.write(commands.getoutput(cmd))
157 176 sys.stdout.flush()
158 177
159 178 # Store certain global objects that IPython modifies
160 179 _displayhook = sys.displayhook
161 180 _excepthook = sys.excepthook
162 181 _main = sys.modules.get('__main__')
163 182
164 183 argv = default_argv()
165 184
166 185 # Start IPython instance. We customize it to start with minimal frills.
167 186 user_ns,global_ns = IPython.ipapi.make_user_namespaces(ipnsdict(),dict())
168 187 IPython.Shell.IPShell(argv,user_ns,global_ns)
169 188
170 189 # Deactivate the various python system hooks added by ipython for
171 190 # interactive convenience so we don't confuse the doctest system
172 191 sys.modules['__main__'] = _main
173 192 sys.displayhook = _displayhook
174 193 sys.excepthook = _excepthook
175 194
176 195 # So that ipython magics and aliases can be doctested (they work by making
177 196 # a call into a global _ip object)
178 197 _ip = IPython.ipapi.get()
179 198 __builtin__._ip = _ip
180 199
181 200 # Modify the IPython system call with one that uses getoutput, so that we
182 201 # can capture subcommands and print them to Python's stdout, otherwise the
183 202 # doctest machinery would miss them.
184 203 _ip.system = xsys
185 204
186 205 # Also patch our %run function in.
187 206 im = new.instancemethod(_run_ns_sync,_ip.IP, _ip.IP.__class__)
188 207 _ip.IP.magic_run_ori = _ip.IP.magic_run
189 208 _ip.IP.magic_run = im
190 209
191 210 # The start call MUST be made here. I'm not sure yet why it doesn't work if
192 211 # it is made later, at plugin initialization time, but in all my tests, that's
193 212 # the case.
194 213 start_ipython()
195 214
196 215 # *** END HACK ***
197 216 ###########################################################################
198 217
199 218 # Classes and functions
200 219
201 220 def is_extension_module(filename):
202 221 """Return whether the given filename is an extension module.
203 222
204 223 This simply checks that the extension is either .so or .pyd.
205 224 """
206 225 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
207 226
208 227
209 228 class DocTestSkip(object):
210 229 """Object wrapper for doctests to be skipped."""
211 230
212 231 ds_skip = """Doctest to skip.
213 232 >>> 1 #doctest: +SKIP
214 233 """
215 234
216 235 def __init__(self,obj):
217 236 self.obj = obj
218 237
219 238 def __getattribute__(self,key):
220 239 if key == '__doc__':
221 240 return DocTestSkip.ds_skip
222 241 else:
223 242 return getattr(object.__getattribute__(self,'obj'),key)
224 243
225 244 # Modified version of the one in the stdlib, that fixes a python bug (doctests
226 245 # not found in extension modules, http://bugs.python.org/issue3158)
227 246 class DocTestFinder(doctest.DocTestFinder):
228 247
229 248 def _from_module(self, module, object):
230 249 """
231 250 Return true if the given object is defined in the given
232 251 module.
233 252 """
234 253 if module is None:
235 254 return True
236 255 elif inspect.isfunction(object):
237 256 return module.__dict__ is object.func_globals
238 257 elif inspect.isbuiltin(object):
239 258 return module.__name__ == object.__module__
240 259 elif inspect.isclass(object):
241 260 return module.__name__ == object.__module__
242 261 elif inspect.ismethod(object):
243 262 # This one may be a bug in cython that fails to correctly set the
244 263 # __module__ attribute of methods, but since the same error is easy
245 264 # to make by extension code writers, having this safety in place
246 265 # isn't such a bad idea
247 266 return module.__name__ == object.im_class.__module__
248 267 elif inspect.getmodule(object) is not None:
249 268 return module is inspect.getmodule(object)
250 269 elif hasattr(object, '__module__'):
251 270 return module.__name__ == object.__module__
252 271 elif isinstance(object, property):
253 272 return True # [XX] no way not be sure.
254 273 else:
255 274 raise ValueError("object must be a class or function")
256 275
257 276 def _find(self, tests, obj, name, module, source_lines, globs, seen):
258 277 """
259 278 Find tests for the given object and any contained objects, and
260 279 add them to `tests`.
261 280 """
262 281
263 282 if hasattr(obj,"skip_doctest"):
264 283 #print 'SKIPPING DOCTEST FOR:',obj # dbg
265 284 obj = DocTestSkip(obj)
266 285
267 286 doctest.DocTestFinder._find(self,tests, obj, name, module,
268 287 source_lines, globs, seen)
269 288
270 289 # Below we re-run pieces of the above method with manual modifications,
271 290 # because the original code is buggy and fails to correctly identify
272 291 # doctests in extension modules.
273 292
274 293 # Local shorthands
275 294 from inspect import isroutine, isclass, ismodule
276 295
277 296 # Look for tests in a module's contained objects.
278 297 if inspect.ismodule(obj) and self._recurse:
279 298 for valname, val in obj.__dict__.items():
280 299 valname1 = '%s.%s' % (name, valname)
281 300 if ( (isroutine(val) or isclass(val))
282 301 and self._from_module(module, val) ):
283 302
284 303 self._find(tests, val, valname1, module, source_lines,
285 304 globs, seen)
286 305
287 306 # Look for tests in a class's contained objects.
288 307 if inspect.isclass(obj) and self._recurse:
289 308 #print 'RECURSE into class:',obj # dbg
290 309 for valname, val in obj.__dict__.items():
291 310 # Special handling for staticmethod/classmethod.
292 311 if isinstance(val, staticmethod):
293 312 val = getattr(obj, valname)
294 313 if isinstance(val, classmethod):
295 314 val = getattr(obj, valname).im_func
296 315
297 316 # Recurse to methods, properties, and nested classes.
298 317 if ((inspect.isfunction(val) or inspect.isclass(val) or
299 318 inspect.ismethod(val) or
300 319 isinstance(val, property)) and
301 320 self._from_module(module, val)):
302 321 valname = '%s.%s' % (name, valname)
303 322 self._find(tests, val, valname, module, source_lines,
304 323 globs, seen)
305 324
306 325
307 326 class IPDoctestOutputChecker(doctest.OutputChecker):
308 327 """Second-chance checker with support for random tests.
309 328
310 329 If the default comparison doesn't pass, this checker looks in the expected
311 330 output string for flags that tell us to ignore the output.
312 331 """
313 332
314 333 random_re = re.compile(r'#\s*random\s+')
315 334
316 335 def check_output(self, want, got, optionflags):
317 336 """Check output, accepting special markers embedded in the output.
318 337
319 338 If the output didn't pass the default validation but the special string
320 339 '#random' is included, we accept it."""
321 340
322 341 # Let the original tester verify first, in case people have valid tests
323 342 # that happen to have a comment saying '#random' embedded in.
324 343 ret = doctest.OutputChecker.check_output(self, want, got,
325 344 optionflags)
326 345 if not ret and self.random_re.search(want):
327 346 #print >> sys.stderr, 'RANDOM OK:',want # dbg
328 347 return True
329 348
330 349 return ret
331 350
332 351
333 352 class DocTestCase(doctests.DocTestCase):
334 353 """Proxy for DocTestCase: provides an address() method that
335 354 returns the correct address for the doctest case. Otherwise
336 355 acts as a proxy to the test case. To provide hints for address(),
337 356 an obj may also be passed -- this will be used as the test object
338 357 for purposes of determining the test address, if it is provided.
339 358 """
340 359
341 360 # Note: this method was taken from numpy's nosetester module.
342 361
343 362 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
344 363 # its constructor that blocks non-default arguments from being passed
345 364 # down into doctest.DocTestCase
346 365
347 366 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
348 367 checker=None, obj=None, result_var='_'):
349 368 self._result_var = result_var
350 369 doctests.DocTestCase.__init__(self, test,
351 370 optionflags=optionflags,
352 371 setUp=setUp, tearDown=tearDown,
353 372 checker=checker)
354 373 # Now we must actually copy the original constructor from the stdlib
355 374 # doctest class, because we can't call it directly and a bug in nose
356 375 # means it never gets passed the right arguments.
357 376
358 377 self._dt_optionflags = optionflags
359 378 self._dt_checker = checker
360 379 self._dt_test = test
361 380 self._dt_setUp = setUp
362 381 self._dt_tearDown = tearDown
363 382
364 383 # XXX - store this runner once in the object!
365 384 runner = IPDocTestRunner(optionflags=optionflags,
366 385 checker=checker, verbose=False)
367 386 self._dt_runner = runner
368 387
369 388
370 389 # Each doctest should remember what directory it was loaded from...
371 390 self._ori_dir = os.getcwd()
372 391
373 392 # Modified runTest from the default stdlib
374 393 def runTest(self):
375 394 test = self._dt_test
376 395 runner = self._dt_runner
377 396
378 397 old = sys.stdout
379 398 new = StringIO()
380 399 optionflags = self._dt_optionflags
381 400
382 401 if not (optionflags & REPORTING_FLAGS):
383 402 # The option flags don't include any reporting flags,
384 403 # so add the default reporting flags
385 404 optionflags |= _unittest_reportflags
386 405
387 406 try:
388 407 # Save our current directory and switch out to the one where the
389 408 # test was originally created, in case another doctest did a
390 409 # directory change. We'll restore this in the finally clause.
391 410 curdir = os.getcwd()
392 411 os.chdir(self._ori_dir)
393 412
394 413 runner.DIVIDER = "-"*70
395 414 failures, tries = runner.run(test,out=new.write,
396 415 clear_globs=False)
397 416 finally:
398 417 sys.stdout = old
399 418 os.chdir(curdir)
400 419
401 420 if failures:
402 421 raise self.failureException(self.format_failure(new.getvalue()))
403 422
404 423 def setUp(self):
405 424 """Modified test setup that syncs with ipython namespace"""
406 425
407 426 if isinstance(self._dt_test.examples[0],IPExample):
408 427 # for IPython examples *only*, we swap the globals with the ipython
409 428 # namespace, after updating it with the globals (which doctest
410 429 # fills with the necessary info from the module being tested).
411 430 _ip.IP.user_ns.update(self._dt_test.globs)
412 431 self._dt_test.globs = _ip.IP.user_ns
413 432
414 433 doctests.DocTestCase.setUp(self)
415 434
416 435
417 436 # A simple subclassing of the original with a different class name, so we can
418 437 # distinguish and treat differently IPython examples from pure python ones.
419 438 class IPExample(doctest.Example): pass
420 439
421 440
422 441 class IPExternalExample(doctest.Example):
423 442 """Doctest examples to be run in an external process."""
424 443
425 444 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
426 445 options=None):
427 446 # Parent constructor
428 447 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
429 448
430 449 # An EXTRA newline is needed to prevent pexpect hangs
431 450 self.source += '\n'
432 451
433 452
434 453 class IPDocTestParser(doctest.DocTestParser):
435 454 """
436 455 A class used to parse strings containing doctest examples.
437 456
438 457 Note: This is a version modified to properly recognize IPython input and
439 458 convert any IPython examples into valid Python ones.
440 459 """
441 460 # This regular expression is used to find doctest examples in a
442 461 # string. It defines three groups: `source` is the source code
443 462 # (including leading indentation and prompts); `indent` is the
444 463 # indentation of the first (PS1) line of the source code; and
445 464 # `want` is the expected output (including leading indentation).
446 465
447 466 # Classic Python prompts or default IPython ones
448 467 _PS1_PY = r'>>>'
449 468 _PS2_PY = r'\.\.\.'
450 469
451 470 _PS1_IP = r'In\ \[\d+\]:'
452 471 _PS2_IP = r'\ \ \ \.\.\.+:'
453 472
454 473 _RE_TPL = r'''
455 474 # Source consists of a PS1 line followed by zero or more PS2 lines.
456 475 (?P<source>
457 476 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
458 477 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
459 478 \n? # a newline
460 479 # Want consists of any non-blank lines that do not start with PS1.
461 480 (?P<want> (?:(?![ ]*$) # Not a blank line
462 481 (?![ ]*%s) # Not a line starting with PS1
463 482 (?![ ]*%s) # Not a line starting with PS2
464 483 .*$\n? # But any other line
465 484 )*)
466 485 '''
467 486
468 487 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
469 488 re.MULTILINE | re.VERBOSE)
470 489
471 490 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
472 491 re.MULTILINE | re.VERBOSE)
473 492
474 493 # Mark a test as being fully random. In this case, we simply append the
475 494 # random marker ('#random') to each individual example's output. This way
476 495 # we don't need to modify any other code.
477 496 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
478 497
479 498 # Mark tests to be executed in an external process - currently unsupported.
480 499 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
481 500
482 501 def ip2py(self,source):
483 502 """Convert input IPython source into valid Python."""
484 503 out = []
485 504 newline = out.append
486 505 #print 'IPSRC:\n',source,'\n###' # dbg
487 506 # The input source must be first stripped of all bracketing whitespace
488 507 # and turned into lines, so it looks to the parser like regular user
489 508 # input
490 509 for lnum,line in enumerate(source.strip().splitlines()):
491 510 newline(_ip.IP.prefilter(line,lnum>0))
492 511 newline('') # ensure a closing newline, needed by doctest
493 512 #print "PYSRC:", '\n'.join(out) # dbg
494 513 return '\n'.join(out)
495 514
496 515 def parse(self, string, name='<string>'):
497 516 """
498 517 Divide the given string into examples and intervening text,
499 518 and return them as a list of alternating Examples and strings.
500 519 Line numbers for the Examples are 0-based. The optional
501 520 argument `name` is a name identifying this string, and is only
502 521 used for error messages.
503 522 """
504 523
505 524 #print 'Parse string:\n',string # dbg
506 525
507 526 string = string.expandtabs()
508 527 # If all lines begin with the same indentation, then strip it.
509 528 min_indent = self._min_indent(string)
510 529 if min_indent > 0:
511 530 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
512 531
513 532 output = []
514 533 charno, lineno = 0, 0
515 534
516 535 # We make 'all random' tests by adding the '# random' mark to every
517 536 # block of output in the test.
518 537 if self._RANDOM_TEST.search(string):
519 538 random_marker = '\n# random'
520 539 else:
521 540 random_marker = ''
522 541
523 542 # Whether to convert the input from ipython to python syntax
524 543 ip2py = False
525 544 # Find all doctest examples in the string. First, try them as Python
526 545 # examples, then as IPython ones
527 546 terms = list(self._EXAMPLE_RE_PY.finditer(string))
528 547 if terms:
529 548 # Normal Python example
530 549 #print '-'*70 # dbg
531 550 #print 'PyExample, Source:\n',string # dbg
532 551 #print '-'*70 # dbg
533 552 Example = doctest.Example
534 553 else:
535 554 # It's an ipython example. Note that IPExamples are run
536 555 # in-process, so their syntax must be turned into valid python.
537 556 # IPExternalExamples are run out-of-process (via pexpect) so they
538 557 # don't need any filtering (a real ipython will be executing them).
539 558 terms = list(self._EXAMPLE_RE_IP.finditer(string))
540 559 if self._EXTERNAL_IP.search(string):
541 560 #print '-'*70 # dbg
542 561 #print 'IPExternalExample, Source:\n',string # dbg
543 562 #print '-'*70 # dbg
544 563 Example = IPExternalExample
545 564 else:
546 565 #print '-'*70 # dbg
547 566 #print 'IPExample, Source:\n',string # dbg
548 567 #print '-'*70 # dbg
549 568 Example = IPExample
550 569 ip2py = True
551 570
552 571 for m in terms:
553 572 # Add the pre-example text to `output`.
554 573 output.append(string[charno:m.start()])
555 574 # Update lineno (lines before this example)
556 575 lineno += string.count('\n', charno, m.start())
557 576 # Extract info from the regexp match.
558 577 (source, options, want, exc_msg) = \
559 578 self._parse_example(m, name, lineno,ip2py)
560 579
561 580 # Append the random-output marker (it defaults to empty in most
562 581 # cases, it's only non-empty for 'all-random' tests):
563 582 want += random_marker
564 583
565 584 if Example is IPExternalExample:
566 585 options[doctest.NORMALIZE_WHITESPACE] = True
567 586 want += '\n'
568 587
569 588 # Create an Example, and add it to the list.
570 589 if not self._IS_BLANK_OR_COMMENT(source):
571 590 output.append(Example(source, want, exc_msg,
572 591 lineno=lineno,
573 592 indent=min_indent+len(m.group('indent')),
574 593 options=options))
575 594 # Update lineno (lines inside this example)
576 595 lineno += string.count('\n', m.start(), m.end())
577 596 # Update charno.
578 597 charno = m.end()
579 598 # Add any remaining post-example text to `output`.
580 599 output.append(string[charno:])
581 600 return output
582 601
583 602 def _parse_example(self, m, name, lineno,ip2py=False):
584 603 """
585 604 Given a regular expression match from `_EXAMPLE_RE` (`m`),
586 605 return a pair `(source, want)`, where `source` is the matched
587 606 example's source code (with prompts and indentation stripped);
588 607 and `want` is the example's expected output (with indentation
589 608 stripped).
590 609
591 610 `name` is the string's name, and `lineno` is the line number
592 611 where the example starts; both are used for error messages.
593 612
594 613 Optional:
595 614 `ip2py`: if true, filter the input via IPython to convert the syntax
596 615 into valid python.
597 616 """
598 617
599 618 # Get the example's indentation level.
600 619 indent = len(m.group('indent'))
601 620
602 621 # Divide source into lines; check that they're properly
603 622 # indented; and then strip their indentation & prompts.
604 623 source_lines = m.group('source').split('\n')
605 624
606 625 # We're using variable-length input prompts
607 626 ps1 = m.group('ps1')
608 627 ps2 = m.group('ps2')
609 628 ps1_len = len(ps1)
610 629
611 630 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
612 631 if ps2:
613 632 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
614 633
615 634 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
616 635
617 636 if ip2py:
618 637 # Convert source input from IPython into valid Python syntax
619 638 source = self.ip2py(source)
620 639
621 640 # Divide want into lines; check that it's properly indented; and
622 641 # then strip the indentation. Spaces before the last newline should
623 642 # be preserved, so plain rstrip() isn't good enough.
624 643 want = m.group('want')
625 644 want_lines = want.split('\n')
626 645 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
627 646 del want_lines[-1] # forget final newline & spaces after it
628 647 self._check_prefix(want_lines, ' '*indent, name,
629 648 lineno + len(source_lines))
630 649
631 650 # Remove ipython output prompt that might be present in the first line
632 651 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
633 652
634 653 want = '\n'.join([wl[indent:] for wl in want_lines])
635 654
636 655 # If `want` contains a traceback message, then extract it.
637 656 m = self._EXCEPTION_RE.match(want)
638 657 if m:
639 658 exc_msg = m.group('msg')
640 659 else:
641 660 exc_msg = None
642 661
643 662 # Extract options from the source.
644 663 options = self._find_options(source, name, lineno)
645 664
646 665 return source, options, want, exc_msg
647 666
648 667 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
649 668 """
650 669 Given the lines of a source string (including prompts and
651 670 leading indentation), check to make sure that every prompt is
652 671 followed by a space character. If any line is not followed by
653 672 a space character, then raise ValueError.
654 673
655 674 Note: IPython-modified version which takes the input prompt length as a
656 675 parameter, so that prompts of variable length can be dealt with.
657 676 """
658 677 space_idx = indent+ps1_len
659 678 min_len = space_idx+1
660 679 for i, line in enumerate(lines):
661 680 if len(line) >= min_len and line[space_idx] != ' ':
662 681 raise ValueError('line %r of the docstring for %s '
663 682 'lacks blank after %s: %r' %
664 683 (lineno+i+1, name,
665 684 line[indent:space_idx], line))
666 685
667 686
668 687 SKIP = doctest.register_optionflag('SKIP')
669 688
670 689
671 690 class IPDocTestRunner(doctest.DocTestRunner,object):
672 691 """Test runner that synchronizes the IPython namespace with test globals.
673 692 """
674 693
675 694 def run(self, test, compileflags=None, out=None, clear_globs=True):
676 695
677 696 # Hack: ipython needs access to the execution context of the example,
678 697 # so that it can propagate user variables loaded by %run into
679 698 # test.globs. We put them here into our modified %run as a function
680 699 # attribute. Our new %run will then only make the namespace update
681 700 # when called (rather than unconconditionally updating test.globs here
682 701 # for all examples, most of which won't be calling %run anyway).
683 702 _run_ns_sync.test_globs = test.globs
684 703 _run_ns_sync.test_filename = test.filename
685 704
686 705 return super(IPDocTestRunner,self).run(test,
687 706 compileflags,out,clear_globs)
688 707
689 708
690 709 class DocFileCase(doctest.DocFileCase):
691 710 """Overrides to provide filename
692 711 """
693 712 def address(self):
694 713 return (self._dt_test.filename, None, None)
695 714
696 715
697 716 class ExtensionDoctest(doctests.Doctest):
698 717 """Nose Plugin that supports doctests in extension modules.
699 718 """
700 719 name = 'extdoctest' # call nosetests with --with-extdoctest
701 720 enabled = True
702 721
703 722 def __init__(self,exclude_patterns=None):
704 723 """Create a new ExtensionDoctest plugin.
705 724
706 725 Parameters
707 726 ----------
708 727
709 728 exclude_patterns : sequence of strings, optional
710 729 These patterns are compiled as regular expressions, subsequently used
711 730 to exclude any filename which matches them from inclusion in the test
712 731 suite (using pattern.search(), NOT pattern.match() ).
713 732 """
714 733
715 734 if exclude_patterns is None:
716 735 exclude_patterns = []
717 736 self.exclude_patterns = map(re.compile,exclude_patterns)
718 737 doctests.Doctest.__init__(self)
719 738
720 739 def options(self, parser, env=os.environ):
721 740 Plugin.options(self, parser, env)
722 741 parser.add_option('--doctest-tests', action='store_true',
723 742 dest='doctest_tests',
724 743 default=env.get('NOSE_DOCTEST_TESTS',True),
725 744 help="Also look for doctests in test modules. "
726 745 "Note that classes, methods and functions should "
727 746 "have either doctests or non-doctest tests, "
728 747 "not both. [NOSE_DOCTEST_TESTS]")
729 748 parser.add_option('--doctest-extension', action="append",
730 749 dest="doctestExtension",
731 750 help="Also look for doctests in files with "
732 751 "this extension [NOSE_DOCTEST_EXTENSION]")
733 752 # Set the default as a list, if given in env; otherwise
734 753 # an additional value set on the command line will cause
735 754 # an error.
736 755 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
737 756 if env_setting is not None:
738 757 parser.set_defaults(doctestExtension=tolist(env_setting))
739 758
740 759
741 760 def configure(self, options, config):
742 761 Plugin.configure(self, options, config)
743 762 self.doctest_tests = options.doctest_tests
744 763 self.extension = tolist(options.doctestExtension)
745 764
746 765 self.parser = doctest.DocTestParser()
747 766 self.finder = DocTestFinder()
748 767 self.checker = IPDoctestOutputChecker()
749 768 self.globs = None
750 769 self.extraglobs = None
751 770
752 771
753 772 def loadTestsFromExtensionModule(self,filename):
754 773 bpath,mod = os.path.split(filename)
755 774 modname = os.path.splitext(mod)[0]
756 775 try:
757 776 sys.path.append(bpath)
758 777 module = __import__(modname)
759 778 tests = list(self.loadTestsFromModule(module))
760 779 finally:
761 780 sys.path.pop()
762 781 return tests
763 782
764 783 # NOTE: the method below is almost a copy of the original one in nose, with
765 784 # a few modifications to control output checking.
766 785
767 786 def loadTestsFromModule(self, module):
768 787 #print '*** ipdoctest - lTM',module # dbg
769 788
770 789 if not self.matches(module.__name__):
771 790 log.debug("Doctest doesn't want module %s", module)
772 791 return
773 792
774 793 tests = self.finder.find(module,globs=self.globs,
775 794 extraglobs=self.extraglobs)
776 795 if not tests:
777 796 return
778 797
779 798 # always use whitespace and ellipsis options
780 799 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
781 800
782 801 tests.sort()
783 802 module_file = module.__file__
784 803 if module_file[-4:] in ('.pyc', '.pyo'):
785 804 module_file = module_file[:-1]
786 805 for test in tests:
787 806 if not test.examples:
788 807 continue
789 808 if not test.filename:
790 809 test.filename = module_file
791 810
792 811 yield DocTestCase(test,
793 812 optionflags=optionflags,
794 813 checker=self.checker)
795 814
796 815
797 816 def loadTestsFromFile(self, filename):
798 817 if is_extension_module(filename):
799 818 for t in self.loadTestsFromExtensionModule(filename):
800 819 yield t
801 820 else:
802 821 if self.extension and anyp(filename.endswith, self.extension):
803 822 name = os.path.basename(filename)
804 823 dh = open(filename)
805 824 try:
806 825 doc = dh.read()
807 826 finally:
808 827 dh.close()
809 828 test = self.parser.get_doctest(
810 829 doc, globs={'__file__': filename}, name=name,
811 830 filename=filename, lineno=0)
812 831 if test.examples:
813 832 #print 'FileCase:',test.examples # dbg
814 833 yield DocFileCase(test)
815 834 else:
816 835 yield False # no tests to load
817 836
818 837 def wantFile(self,filename):
819 838 """Return whether the given filename should be scanned for tests.
820 839
821 840 Modified version that accepts extension modules as valid containers for
822 841 doctests.
823 842 """
824 843 #print '*** ipdoctest- wantFile:',filename # dbg
825 844
826 845 for pat in self.exclude_patterns:
827 846 if pat.search(filename):
828 847 #print '###>>> SKIP:',filename # dbg
829 848 return False
830 849
831 850 if is_extension_module(filename):
832 851 return True
833 852 else:
834 853 return doctests.Doctest.wantFile(self,filename)
835 854
836 855
837 856 class IPythonDoctest(ExtensionDoctest):
838 857 """Nose Plugin that supports doctests in extension modules.
839 858 """
840 859 name = 'ipdoctest' # call nosetests with --with-ipdoctest
841 860 enabled = True
842 861
843 862 def makeTest(self, obj, parent):
844 863 """Look for doctests in the given object, which will be a
845 864 function, method or class.
846 865 """
847 866 # always use whitespace and ellipsis options
848 867 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
849 868
850 869 doctests = self.finder.find(obj, module=getmodule(parent))
851 870 if doctests:
852 871 for test in doctests:
853 872 if len(test.examples) == 0:
854 873 continue
855 874
856 875 yield DocTestCase(test, obj=obj,
857 876 optionflags=optionflags,
858 877 checker=self.checker)
859 878
860 879 def options(self, parser, env=os.environ):
861 880 Plugin.options(self, parser, env)
862 881 parser.add_option('--ipdoctest-tests', action='store_true',
863 882 dest='ipdoctest_tests',
864 883 default=env.get('NOSE_IPDOCTEST_TESTS',True),
865 884 help="Also look for doctests in test modules. "
866 885 "Note that classes, methods and functions should "
867 886 "have either doctests or non-doctest tests, "
868 887 "not both. [NOSE_IPDOCTEST_TESTS]")
869 888 parser.add_option('--ipdoctest-extension', action="append",
870 889 dest="ipdoctest_extension",
871 890 help="Also look for doctests in files with "
872 891 "this extension [NOSE_IPDOCTEST_EXTENSION]")
873 892 # Set the default as a list, if given in env; otherwise
874 893 # an additional value set on the command line will cause
875 894 # an error.
876 895 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
877 896 if env_setting is not None:
878 897 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
879 898
880 899 def configure(self, options, config):
881 900 Plugin.configure(self, options, config)
882 901 self.doctest_tests = options.ipdoctest_tests
883 902 self.extension = tolist(options.ipdoctest_extension)
884 903
885 904 self.parser = IPDocTestParser()
886 905 self.finder = DocTestFinder(parser=self.parser)
887 906 self.checker = IPDoctestOutputChecker()
888 907 self.globs = None
889 908 self.extraglobs = None
@@ -1,151 +1,235 b''
1 1 """Tests for various magic functions.
2 2
3 3 Needs to be run by nose (to make ipython session available).
4 4 """
5 5
6 6 # Standard library imports
7 7 import os
8 8 import sys
9 import tempfile
10 import types
9 11
10 12 # Third-party imports
11 13 import nose.tools as nt
12 14
13 15 # From our own code
14 16 from IPython.testing import decorators as dec
17 from IPython.testing import tools as tt
15 18
16 19 #-----------------------------------------------------------------------------
17 20 # Test functions begin
18 21
19 22 def test_rehashx():
20 23 # clear up everything
21 24 _ip.IP.alias_table.clear()
22 25 del _ip.db['syscmdlist']
23 26
24 27 _ip.magic('rehashx')
25 28 # Practically ALL ipython development systems will have more than 10 aliases
26 29
27 30 assert len(_ip.IP.alias_table) > 10
28 31 for key, val in _ip.IP.alias_table.items():
29 32 # we must strip dots from alias names
30 33 assert '.' not in key
31 34
32 35 # rehashx must fill up syscmdlist
33 36 scoms = _ip.db['syscmdlist']
34 37 assert len(scoms) > 10
35 38
36 39
37 def doctest_run_ns():
38 """Classes declared %run scripts must be instantiable afterwards.
39
40 In [11]: run tclass foo
41
42 In [12]: isinstance(f(),foo)
43 Out[12]: True
44 """
45
46
47 def doctest_run_ns2():
48 """Classes declared %run scripts must be instantiable afterwards.
49
50 In [4]: run tclass C-first_pass
51
52 In [5]: run tclass C-second_pass
53 tclass.py: deleting object: C-first_pass
54 """
55
56
57 40 def doctest_hist_f():
58 41 """Test %hist -f with temporary filename.
59 42
60 43 In [9]: import tempfile
61 44
62 45 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
63 46
64 47 In [11]: %history -n -f $tfile 3
65 48 """
66 49
67 50
68 51 def doctest_hist_r():
69 52 """Test %hist -r
70 53
71 54 XXX - This test is not recording the output correctly. Not sure why...
72 55
73 56 In [6]: x=1
74 57
75 58 In [7]: hist -n -r 2
76 59 x=1 # random
77 60 hist -n -r 2 # random
78 61 """
79 62
80 63
81 64 def test_obj_del():
82 65 """Test that object's __del__ methods are called on exit."""
83 66 test_dir = os.path.dirname(__file__)
84 67 del_file = os.path.join(test_dir,'obj_del.py')
85 68 out = _ip.IP.getoutput('ipython %s' % del_file)
86 69 nt.assert_equals(out,'obj_del.py: object A deleted')
87 70
88 71
89 72 def test_shist():
90 73 # Simple tests of ShadowHist class - test generator.
91 74 import os, shutil, tempfile
92 75
93 76 from IPython.Extensions import pickleshare
94 77 from IPython.history import ShadowHist
95 78
96 79 tfile = tempfile.mktemp('','tmp-ipython-')
97 80
98 81 db = pickleshare.PickleShareDB(tfile)
99 82 s = ShadowHist(db)
100 83 s.add('hello')
101 84 s.add('world')
102 85 s.add('hello')
103 86 s.add('hello')
104 87 s.add('karhu')
105 88
106 89 yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
107 90
108 91 yield nt.assert_equal,s.get(2),'world'
109 92
110 93 shutil.rmtree(tfile)
111 94
112 95 @dec.skipif_not_numpy
113 96 def test_numpy_clear_array_undec():
114 97 _ip.ex('import numpy as np')
115 98 _ip.ex('a = np.empty(2)')
116 99
117 100 yield nt.assert_true,'a' in _ip.user_ns
118 101 _ip.magic('clear array')
119 102 yield nt.assert_false,'a' in _ip.user_ns
120 103
121 104
122 105 @dec.skip()
123 106 def test_fail_dec(*a,**k):
124 107 yield nt.assert_true, False
125 108
126 109 @dec.skip('This one shouldn not run')
127 110 def test_fail_dec2(*a,**k):
128 111 yield nt.assert_true, False
129 112
130 113 @dec.skipknownfailure
131 114 def test_fail_dec3(*a,**k):
132 115 yield nt.assert_true, False
133 116
134 117
135 118 def doctest_refbug():
136 119 """Very nasty problem with references held by multiple runs of a script.
137 120 See: https://bugs.launchpad.net/ipython/+bug/269966
138 121
139 122 In [1]: _ip.IP.clear_main_mod_cache()
140 123
141 124 In [2]: run refbug
142 125
143 126 In [3]: call_f()
144 127 lowercased: hello
145 128
146 129 In [4]: run refbug
147 130
148 131 In [5]: call_f()
149 132 lowercased: hello
150 133 lowercased: hello
151 134 """
135
136 #-----------------------------------------------------------------------------
137 # Tests for %run
138 #-----------------------------------------------------------------------------
139
140 # %run is critical enough that it's a good idea to have a solid collection of
141 # tests for it, some as doctests and some as normal tests.
142
143 def doctest_run_ns():
144 """Classes declared %run scripts must be instantiable afterwards.
145
146 In [11]: run tclass foo
147
148 In [12]: isinstance(f(),foo)
149 Out[12]: True
150 """
151
152
153 def doctest_run_ns2():
154 """Classes declared %run scripts must be instantiable afterwards.
155
156 In [4]: run tclass C-first_pass
157
158 In [5]: run tclass C-second_pass
159 tclass.py: deleting object: C-first_pass
160 """
161
162 def doctest_run_builtins():
163 """Check that %run doesn't damage __builtins__ via a doctest.
164
165 This is similar to the test_run_builtins, but I want *both* forms of the
166 test to catch any possible glitches in our testing machinery, since that
167 modifies %run somewhat. So for this, we have both a normal test (below)
168 and a doctest (this one).
169
170 In [1]: import tempfile
171
172 In [2]: bid1 = id(__builtins__)
173
174 In [3]: f = tempfile.NamedTemporaryFile()
175
176 In [4]: f.write('pass\\n')
177
178 In [5]: f.flush()
179
180 In [6]: print 'B1:',type(__builtins__)
181 B1: <type 'module'>
182
183 In [7]: %run $f.name
184
185 In [8]: bid2 = id(__builtins__)
186
187 In [9]: print 'B2:',type(__builtins__)
188 B2: <type 'module'>
189
190 In [10]: bid1 == bid2
191 Out[10]: True
192 """
193
194 # For some tests, it will be handy to organize them in a class with a common
195 # setup that makes a temp file
196
197 class TestMagicRun(object):
198
199 def setup(self):
200 """Make a valid python temp file."""
201 f = tempfile.NamedTemporaryFile()
202 f.write('pass\n')
203 f.flush()
204 self.tmpfile = f
205
206 def run_tmpfile(self):
207 _ip.magic('run %s' % self.tmpfile.name)
208
209 def test_builtins_id(self):
210 """Check that %run doesn't damage __builtins__ """
211
212 # Test that the id of __builtins__ is not modified by %run
213 bid1 = id(_ip.user_ns['__builtins__'])
214 self.run_tmpfile()
215 bid2 = id(_ip.user_ns['__builtins__'])
216 tt.assert_equals(bid1, bid2)
217
218 def test_builtins_type(self):
219 """Check that the type of __builtins__ doesn't change with %run.
220
221 However, the above could pass if __builtins__ was already modified to
222 be a dict (it should be a module) by a previous use of %run. So we
223 also check explicitly that it really is a module:
224 """
225 self.run_tmpfile()
226 tt.assert_equals(type(_ip.user_ns['__builtins__']),type(sys))
227
228 def test_prompts(self):
229 """Test that prompts correctly generate after %run"""
230 self.run_tmpfile()
231 p2 = str(_ip.IP.outputcache.prompt2).strip()
232 nt.assert_equals(p2[:3], '...')
233
234 def teardown(self):
235 self.tmpfile.close()
General Comments 0
You need to be logged in to leave comments. Login now