##// END OF EJS Templates
Add tests to ensure that %run does not modify __builtins__...
Fernando Perez -
Show More
@@ -1,622 +1,626 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Classes for handling input/output prompts.
4 4 """
5 5
6 6 #*****************************************************************************
7 7 # Copyright (C) 2008-2009 The IPython Development Team
8 8 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #*****************************************************************************
13 13
14 14 #****************************************************************************
15 15 # Required modules
16 16 import __builtin__
17 17 import os
18 18 import socket
19 19 import sys
20 20 import time
21 21
22 22 # IPython's own
23 23 from IPython import ColorANSI
24 24 from IPython import Release
25 25 from IPython.external.Itpl import ItplNS
26 26 from IPython.ipapi import TryNext
27 27 from IPython.ipstruct import Struct
28 28 from IPython.macro import Macro
29 29
30 30 from IPython.genutils import *
31 31
32 32 #****************************************************************************
33 33 #Color schemes for Prompts.
34 34
35 35 PromptColors = ColorANSI.ColorSchemeTable()
36 36 InputColors = ColorANSI.InputTermColors # just a shorthand
37 37 Colors = ColorANSI.TermColors # just a shorthand
38 38
39 39 PromptColors.add_scheme(ColorANSI.ColorScheme(
40 40 'NoColor',
41 41 in_prompt = InputColors.NoColor, # Input prompt
42 42 in_number = InputColors.NoColor, # Input prompt number
43 43 in_prompt2 = InputColors.NoColor, # Continuation prompt
44 44 in_normal = InputColors.NoColor, # color off (usu. Colors.Normal)
45 45
46 46 out_prompt = Colors.NoColor, # Output prompt
47 47 out_number = Colors.NoColor, # Output prompt number
48 48
49 49 normal = Colors.NoColor # color off (usu. Colors.Normal)
50 50 ))
51 51
52 52 # make some schemes as instances so we can copy them for modification easily:
53 53 __PColLinux = ColorANSI.ColorScheme(
54 54 'Linux',
55 55 in_prompt = InputColors.Green,
56 56 in_number = InputColors.LightGreen,
57 57 in_prompt2 = InputColors.Green,
58 58 in_normal = InputColors.Normal, # color off (usu. Colors.Normal)
59 59
60 60 out_prompt = Colors.Red,
61 61 out_number = Colors.LightRed,
62 62
63 63 normal = Colors.Normal
64 64 )
65 65 # Don't forget to enter it into the table!
66 66 PromptColors.add_scheme(__PColLinux)
67 67
68 68 # Slightly modified Linux for light backgrounds
69 69 __PColLightBG = __PColLinux.copy('LightBG')
70 70
71 71 __PColLightBG.colors.update(
72 72 in_prompt = InputColors.Blue,
73 73 in_number = InputColors.LightBlue,
74 74 in_prompt2 = InputColors.Blue
75 75 )
76 76 PromptColors.add_scheme(__PColLightBG)
77 77
78 78 del Colors,InputColors
79 79
80 80 #-----------------------------------------------------------------------------
81 81 def multiple_replace(dict, text):
82 82 """ Replace in 'text' all occurences of any key in the given
83 83 dictionary by its corresponding value. Returns the new string."""
84 84
85 85 # Function by Xavier Defrang, originally found at:
86 86 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81330
87 87
88 88 # Create a regular expression from the dictionary keys
89 89 regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
90 90 # For each match, look-up corresponding value in dictionary
91 91 return regex.sub(lambda mo: dict[mo.string[mo.start():mo.end()]], text)
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Special characters that can be used in prompt templates, mainly bash-like
95 95
96 96 # If $HOME isn't defined (Windows), make it an absurd string so that it can
97 97 # never be expanded out into '~'. Basically anything which can never be a
98 98 # reasonable directory name will do, we just want the $HOME -> '~' operation
99 99 # to become a no-op. We pre-compute $HOME here so it's not done on every
100 100 # prompt call.
101 101
102 102 # FIXME:
103 103
104 104 # - This should be turned into a class which does proper namespace management,
105 105 # since the prompt specials need to be evaluated in a certain namespace.
106 106 # Currently it's just globals, which need to be managed manually by code
107 107 # below.
108 108
109 109 # - I also need to split up the color schemes from the prompt specials
110 110 # somehow. I don't have a clean design for that quite yet.
111 111
112 112 HOME = os.environ.get("HOME","//////:::::ZZZZZ,,,~~~")
113 113
114 114 # We precompute a few more strings here for the prompt_specials, which are
115 115 # fixed once ipython starts. This reduces the runtime overhead of computing
116 116 # prompt strings.
117 117 USER = os.environ.get("USER")
118 118 HOSTNAME = socket.gethostname()
119 119 HOSTNAME_SHORT = HOSTNAME.split(".")[0]
120 120 ROOT_SYMBOL = "$#"[os.name=='nt' or os.getuid()==0]
121 121
122 122 prompt_specials_color = {
123 123 # Prompt/history count
124 124 '%n' : '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
125 125 r'\#': '${self.col_num}' '${self.cache.prompt_count}' '${self.col_p}',
126 126 # Just the prompt counter number, WITHOUT any coloring wrappers, so users
127 127 # can get numbers displayed in whatever color they want.
128 128 r'\N': '${self.cache.prompt_count}',
129
129 130 # Prompt/history count, with the actual digits replaced by dots. Used
130 131 # mainly in continuation prompts (prompt_in2)
132 #r'\D': '${"."*len(str(self.cache.prompt_count))}',
133 # More robust form of the above expression, that uses __builtins__
131 134 r'\D': '${"."*__builtins__.len(__builtins__.str(self.cache.prompt_count))}',
135
132 136 # Current working directory
133 137 r'\w': '${os.getcwd()}',
134 138 # Current time
135 139 r'\t' : '${time.strftime("%H:%M:%S")}',
136 140 # Basename of current working directory.
137 141 # (use os.sep to make this portable across OSes)
138 142 r'\W' : '${os.getcwd().split("%s")[-1]}' % os.sep,
139 143 # These X<N> are an extension to the normal bash prompts. They return
140 144 # N terms of the path, after replacing $HOME with '~'
141 145 r'\X0': '${os.getcwd().replace("%s","~")}' % HOME,
142 146 r'\X1': '${self.cwd_filt(1)}',
143 147 r'\X2': '${self.cwd_filt(2)}',
144 148 r'\X3': '${self.cwd_filt(3)}',
145 149 r'\X4': '${self.cwd_filt(4)}',
146 150 r'\X5': '${self.cwd_filt(5)}',
147 151 # Y<N> are similar to X<N>, but they show '~' if it's the directory
148 152 # N+1 in the list. Somewhat like %cN in tcsh.
149 153 r'\Y0': '${self.cwd_filt2(0)}',
150 154 r'\Y1': '${self.cwd_filt2(1)}',
151 155 r'\Y2': '${self.cwd_filt2(2)}',
152 156 r'\Y3': '${self.cwd_filt2(3)}',
153 157 r'\Y4': '${self.cwd_filt2(4)}',
154 158 r'\Y5': '${self.cwd_filt2(5)}',
155 159 # Hostname up to first .
156 160 r'\h': HOSTNAME_SHORT,
157 161 # Full hostname
158 162 r'\H': HOSTNAME,
159 163 # Username of current user
160 164 r'\u': USER,
161 165 # Escaped '\'
162 166 '\\\\': '\\',
163 167 # Newline
164 168 r'\n': '\n',
165 169 # Carriage return
166 170 r'\r': '\r',
167 171 # Release version
168 172 r'\v': Release.version,
169 173 # Root symbol ($ or #)
170 174 r'\$': ROOT_SYMBOL,
171 175 }
172 176
173 177 # A copy of the prompt_specials dictionary but with all color escapes removed,
174 178 # so we can correctly compute the prompt length for the auto_rewrite method.
175 179 prompt_specials_nocolor = prompt_specials_color.copy()
176 180 prompt_specials_nocolor['%n'] = '${self.cache.prompt_count}'
177 181 prompt_specials_nocolor[r'\#'] = '${self.cache.prompt_count}'
178 182
179 183 # Add in all the InputTermColors color escapes as valid prompt characters.
180 184 # They all get added as \\C_COLORNAME, so that we don't have any conflicts
181 185 # with a color name which may begin with a letter used by any other of the
182 186 # allowed specials. This of course means that \\C will never be allowed for
183 187 # anything else.
184 188 input_colors = ColorANSI.InputTermColors
185 189 for _color in dir(input_colors):
186 190 if _color[0] != '_':
187 191 c_name = r'\C_'+_color
188 192 prompt_specials_color[c_name] = getattr(input_colors,_color)
189 193 prompt_specials_nocolor[c_name] = ''
190 194
191 195 # we default to no color for safety. Note that prompt_specials is a global
192 196 # variable used by all prompt objects.
193 197 prompt_specials = prompt_specials_nocolor
194 198
195 199 #-----------------------------------------------------------------------------
196 200 def str_safe(arg):
197 201 """Convert to a string, without ever raising an exception.
198 202
199 203 If str(arg) fails, <ERROR: ... > is returned, where ... is the exception
200 204 error message."""
201 205
202 206 try:
203 207 out = str(arg)
204 208 except UnicodeError:
205 209 try:
206 210 out = arg.encode('utf_8','replace')
207 211 except Exception,msg:
208 212 # let's keep this little duplication here, so that the most common
209 213 # case doesn't suffer from a double try wrapping.
210 214 out = '<ERROR: %s>' % msg
211 215 except Exception,msg:
212 216 out = '<ERROR: %s>' % msg
213 217 return out
214 218
215 219 class BasePrompt(object):
216 220 """Interactive prompt similar to Mathematica's."""
217 221
218 222 def _get_p_template(self):
219 223 return self._p_template
220 224
221 225 def _set_p_template(self,val):
222 226 self._p_template = val
223 227 self.set_p_str()
224 228
225 229 p_template = property(_get_p_template,_set_p_template,
226 230 doc='Template for prompt string creation')
227 231
228 232 def __init__(self,cache,sep,prompt,pad_left=False):
229 233
230 234 # Hack: we access information about the primary prompt through the
231 235 # cache argument. We need this, because we want the secondary prompt
232 236 # to be aligned with the primary one. Color table info is also shared
233 237 # by all prompt classes through the cache. Nice OO spaghetti code!
234 238 self.cache = cache
235 239 self.sep = sep
236 240
237 241 # regexp to count the number of spaces at the end of a prompt
238 242 # expression, useful for prompt auto-rewriting
239 243 self.rspace = re.compile(r'(\s*)$')
240 244 # Flag to left-pad prompt strings to match the length of the primary
241 245 # prompt
242 246 self.pad_left = pad_left
243 247
244 248 # Set template to create each actual prompt (where numbers change).
245 249 # Use a property
246 250 self.p_template = prompt
247 251 self.set_p_str()
248 252
249 253 def set_p_str(self):
250 254 """ Set the interpolating prompt strings.
251 255
252 256 This must be called every time the color settings change, because the
253 257 prompt_specials global may have changed."""
254 258
255 259 import os,time # needed in locals for prompt string handling
256 260 loc = locals()
257 261 try:
258 262 self.p_str = ItplNS('%s%s%s' %
259 263 ('${self.sep}${self.col_p}',
260 264 multiple_replace(prompt_specials, self.p_template),
261 265 '${self.col_norm}'),self.cache.user_ns,loc)
262 266
263 267 self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
264 268 self.p_template),
265 269 self.cache.user_ns,loc)
266 270 except:
267 271 print "Illegal prompt template (check $ usage!):",self.p_template
268 272 self.p_str = self.p_template
269 273 self.p_str_nocolor = self.p_template
270 274
271 275 def write(self,msg): # dbg
272 276 sys.stdout.write(msg)
273 277 return ''
274 278
275 279 def __str__(self):
276 280 """Return a string form of the prompt.
277 281
278 282 This for is useful for continuation and output prompts, since it is
279 283 left-padded to match lengths with the primary one (if the
280 284 self.pad_left attribute is set)."""
281 285
282 286 out_str = str_safe(self.p_str)
283 287 if self.pad_left:
284 288 # We must find the amount of padding required to match lengths,
285 289 # taking the color escapes (which are invisible on-screen) into
286 290 # account.
287 291 esc_pad = len(out_str) - len(str_safe(self.p_str_nocolor))
288 292 format = '%%%ss' % (len(str(self.cache.last_prompt))+esc_pad)
289 293 return format % out_str
290 294 else:
291 295 return out_str
292 296
293 297 # these path filters are put in as methods so that we can control the
294 298 # namespace where the prompt strings get evaluated
295 299 def cwd_filt(self,depth):
296 300 """Return the last depth elements of the current working directory.
297 301
298 302 $HOME is always replaced with '~'.
299 303 If depth==0, the full path is returned."""
300 304
301 305 cwd = os.getcwd().replace(HOME,"~")
302 306 out = os.sep.join(cwd.split(os.sep)[-depth:])
303 307 if out:
304 308 return out
305 309 else:
306 310 return os.sep
307 311
308 312 def cwd_filt2(self,depth):
309 313 """Return the last depth elements of the current working directory.
310 314
311 315 $HOME is always replaced with '~'.
312 316 If depth==0, the full path is returned."""
313 317
314 318 full_cwd = os.getcwd()
315 319 cwd = full_cwd.replace(HOME,"~").split(os.sep)
316 320 if '~' in cwd and len(cwd) == depth+1:
317 321 depth += 1
318 322 drivepart = ''
319 323 if sys.platform == 'win32' and len(cwd) > depth:
320 324 drivepart = os.path.splitdrive(full_cwd)[0]
321 325 out = drivepart + '/'.join(cwd[-depth:])
322 326
323 327 if out:
324 328 return out
325 329 else:
326 330 return os.sep
327 331
328 332 def __nonzero__(self):
329 333 """Implement boolean behavior.
330 334
331 335 Checks whether the p_str attribute is non-empty"""
332 336
333 337 return bool(self.p_template)
334 338
335 339 class Prompt1(BasePrompt):
336 340 """Input interactive prompt similar to Mathematica's."""
337 341
338 342 def __init__(self,cache,sep='\n',prompt='In [\\#]: ',pad_left=True):
339 343 BasePrompt.__init__(self,cache,sep,prompt,pad_left)
340 344
341 345 def set_colors(self):
342 346 self.set_p_str()
343 347 Colors = self.cache.color_table.active_colors # shorthand
344 348 self.col_p = Colors.in_prompt
345 349 self.col_num = Colors.in_number
346 350 self.col_norm = Colors.in_normal
347 351 # We need a non-input version of these escapes for the '--->'
348 352 # auto-call prompts used in the auto_rewrite() method.
349 353 self.col_p_ni = self.col_p.replace('\001','').replace('\002','')
350 354 self.col_norm_ni = Colors.normal
351 355
352 356 def __str__(self):
353 357 self.cache.prompt_count += 1
354 358 self.cache.last_prompt = str_safe(self.p_str_nocolor).split('\n')[-1]
355 359 return str_safe(self.p_str)
356 360
357 361 def auto_rewrite(self):
358 362 """Print a string of the form '--->' which lines up with the previous
359 363 input string. Useful for systems which re-write the user input when
360 364 handling automatically special syntaxes."""
361 365
362 366 curr = str(self.cache.last_prompt)
363 367 nrspaces = len(self.rspace.search(curr).group())
364 368 return '%s%s>%s%s' % (self.col_p_ni,'-'*(len(curr)-nrspaces-1),
365 369 ' '*nrspaces,self.col_norm_ni)
366 370
367 371 class PromptOut(BasePrompt):
368 372 """Output interactive prompt similar to Mathematica's."""
369 373
370 374 def __init__(self,cache,sep='',prompt='Out[\\#]: ',pad_left=True):
371 375 BasePrompt.__init__(self,cache,sep,prompt,pad_left)
372 376 if not self.p_template:
373 377 self.__str__ = lambda: ''
374 378
375 379 def set_colors(self):
376 380 self.set_p_str()
377 381 Colors = self.cache.color_table.active_colors # shorthand
378 382 self.col_p = Colors.out_prompt
379 383 self.col_num = Colors.out_number
380 384 self.col_norm = Colors.normal
381 385
382 386 class Prompt2(BasePrompt):
383 387 """Interactive continuation prompt."""
384 388
385 389 def __init__(self,cache,prompt=' .\\D.: ',pad_left=True):
386 390 self.cache = cache
387 391 self.p_template = prompt
388 392 self.pad_left = pad_left
389 393 self.set_p_str()
390 394
391 395 def set_p_str(self):
392 396 import os,time # needed in locals for prompt string handling
393 397 loc = locals()
394 398 self.p_str = ItplNS('%s%s%s' %
395 399 ('${self.col_p2}',
396 400 multiple_replace(prompt_specials, self.p_template),
397 401 '$self.col_norm'),
398 402 self.cache.user_ns,loc)
399 403 self.p_str_nocolor = ItplNS(multiple_replace(prompt_specials_nocolor,
400 404 self.p_template),
401 405 self.cache.user_ns,loc)
402 406
403 407 def set_colors(self):
404 408 self.set_p_str()
405 409 Colors = self.cache.color_table.active_colors
406 410 self.col_p2 = Colors.in_prompt2
407 411 self.col_norm = Colors.in_normal
408 412 # FIXME (2004-06-16) HACK: prevent crashes for users who haven't
409 413 # updated their prompt_in2 definitions. Remove eventually.
410 414 self.col_p = Colors.out_prompt
411 415 self.col_num = Colors.out_number
412 416
413 417
414 418 #-----------------------------------------------------------------------------
415 419 class CachedOutput:
416 420 """Class for printing output from calculations while keeping a cache of
417 421 reults. It dynamically creates global variables prefixed with _ which
418 422 contain these results.
419 423
420 424 Meant to be used as a sys.displayhook replacement, providing numbered
421 425 prompts and cache services.
422 426
423 427 Initialize with initial and final values for cache counter (this defines
424 428 the maximum size of the cache."""
425 429
426 430 def __init__(self,shell,cache_size,Pprint,
427 431 colors='NoColor',input_sep='\n',
428 432 output_sep='\n',output_sep2='',
429 433 ps1 = None, ps2 = None,ps_out = None,pad_left=True):
430 434
431 435 cache_size_min = 3
432 436 if cache_size <= 0:
433 437 self.do_full_cache = 0
434 438 cache_size = 0
435 439 elif cache_size < cache_size_min:
436 440 self.do_full_cache = 0
437 441 cache_size = 0
438 442 warn('caching was disabled (min value for cache size is %s).' %
439 443 cache_size_min,level=3)
440 444 else:
441 445 self.do_full_cache = 1
442 446
443 447 self.cache_size = cache_size
444 448 self.input_sep = input_sep
445 449
446 450 # we need a reference to the user-level namespace
447 451 self.shell = shell
448 452 self.user_ns = shell.user_ns
449 453 # and to the user's input
450 454 self.input_hist = shell.input_hist
451 455 # and to the user's logger, for logging output
452 456 self.logger = shell.logger
453 457
454 458 # Set input prompt strings and colors
455 459 if cache_size == 0:
456 460 if ps1.find('%n') > -1 or ps1.find(r'\#') > -1 \
457 461 or ps1.find(r'\N') > -1:
458 462 ps1 = '>>> '
459 463 if ps2.find('%n') > -1 or ps2.find(r'\#') > -1 \
460 464 or ps2.find(r'\N') > -1:
461 465 ps2 = '... '
462 466 self.ps1_str = self._set_prompt_str(ps1,'In [\\#]: ','>>> ')
463 467 self.ps2_str = self._set_prompt_str(ps2,' .\\D.: ','... ')
464 468 self.ps_out_str = self._set_prompt_str(ps_out,'Out[\\#]: ','')
465 469
466 470 self.color_table = PromptColors
467 471 self.prompt1 = Prompt1(self,sep=input_sep,prompt=self.ps1_str,
468 472 pad_left=pad_left)
469 473 self.prompt2 = Prompt2(self,prompt=self.ps2_str,pad_left=pad_left)
470 474 self.prompt_out = PromptOut(self,sep='',prompt=self.ps_out_str,
471 475 pad_left=pad_left)
472 476 self.set_colors(colors)
473 477
474 478 # other more normal stuff
475 479 # b/c each call to the In[] prompt raises it by 1, even the first.
476 480 self.prompt_count = 0
477 481 # Store the last prompt string each time, we need it for aligning
478 482 # continuation and auto-rewrite prompts
479 483 self.last_prompt = ''
480 484 self.Pprint = Pprint
481 485 self.output_sep = output_sep
482 486 self.output_sep2 = output_sep2
483 487 self._,self.__,self.___ = '','',''
484 488 self.pprint_types = map(type,[(),[],{}])
485 489
486 490 # these are deliberately global:
487 491 to_user_ns = {'_':self._,'__':self.__,'___':self.___}
488 492 self.user_ns.update(to_user_ns)
489 493
490 494 def _set_prompt_str(self,p_str,cache_def,no_cache_def):
491 495 if p_str is None:
492 496 if self.do_full_cache:
493 497 return cache_def
494 498 else:
495 499 return no_cache_def
496 500 else:
497 501 return p_str
498 502
499 503 def set_colors(self,colors):
500 504 """Set the active color scheme and configure colors for the three
501 505 prompt subsystems."""
502 506
503 507 # FIXME: the prompt_specials global should be gobbled inside this
504 508 # class instead. Do it when cleaning up the whole 3-prompt system.
505 509 global prompt_specials
506 510 if colors.lower()=='nocolor':
507 511 prompt_specials = prompt_specials_nocolor
508 512 else:
509 513 prompt_specials = prompt_specials_color
510 514
511 515 self.color_table.set_active_scheme(colors)
512 516 self.prompt1.set_colors()
513 517 self.prompt2.set_colors()
514 518 self.prompt_out.set_colors()
515 519
516 520 def __call__(self,arg=None):
517 521 """Printing with history cache management.
518 522
519 523 This is invoked everytime the interpreter needs to print, and is
520 524 activated by setting the variable sys.displayhook to it."""
521 525
522 526 # If something injected a '_' variable in __builtin__, delete
523 527 # ipython's automatic one so we don't clobber that. gettext() in
524 528 # particular uses _, so we need to stay away from it.
525 529 if '_' in __builtin__.__dict__:
526 530 try:
527 531 del self.user_ns['_']
528 532 except KeyError:
529 533 pass
530 534 if arg is not None:
531 535 cout_write = Term.cout.write # fast lookup
532 536 # first handle the cache and counters
533 537
534 538 # do not print output if input ends in ';'
535 539 try:
536 540 if self.input_hist[self.prompt_count].endswith(';\n'):
537 541 return
538 542 except IndexError:
539 543 # some uses of ipshellembed may fail here
540 544 pass
541 545 # don't use print, puts an extra space
542 546 cout_write(self.output_sep)
543 547 outprompt = self.shell.hooks.generate_output_prompt()
544 548 if self.do_full_cache:
545 549 cout_write(outprompt)
546 550
547 551 # and now call a possibly user-defined print mechanism
548 552 manipulated_val = self.display(arg)
549 553
550 554 # user display hooks can change the variable to be stored in
551 555 # output history
552 556
553 557 if manipulated_val is not None:
554 558 arg = manipulated_val
555 559
556 560 # avoid recursive reference when displaying _oh/Out
557 561 if arg is not self.user_ns['_oh']:
558 562 self.update(arg)
559 563
560 564 if self.logger.log_output:
561 565 self.logger.log_write(repr(arg),'output')
562 566 cout_write(self.output_sep2)
563 567 Term.cout.flush()
564 568
565 569 def _display(self,arg):
566 570 """Default printer method, uses pprint.
567 571
568 572 Do ip.set_hook("result_display", my_displayhook) for custom result
569 573 display, e.g. when your own objects need special formatting.
570 574 """
571 575 try:
572 576 return IPython.generics.result_display(arg)
573 577 except TryNext:
574 578 return self.shell.hooks.result_display(arg)
575 579
576 580 # Assign the default display method:
577 581 display = _display
578 582
579 583 def update(self,arg):
580 584 #print '***cache_count', self.cache_count # dbg
581 585 if len(self.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
582 586 warn('Output cache limit (currently '+
583 587 `self.cache_size`+' entries) hit.\n'
584 588 'Flushing cache and resetting history counter...\n'
585 589 'The only history variables available will be _,__,___ and _1\n'
586 590 'with the current result.')
587 591
588 592 self.flush()
589 593 # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
590 594 # we cause buggy behavior for things like gettext).
591 595 if '_' not in __builtin__.__dict__:
592 596 self.___ = self.__
593 597 self.__ = self._
594 598 self._ = arg
595 599 self.user_ns.update({'_':self._,'__':self.__,'___':self.___})
596 600
597 601 # hackish access to top-level namespace to create _1,_2... dynamically
598 602 to_main = {}
599 603 if self.do_full_cache:
600 604 new_result = '_'+`self.prompt_count`
601 605 to_main[new_result] = arg
602 606 self.user_ns.update(to_main)
603 607 self.user_ns['_oh'][self.prompt_count] = arg
604 608
605 609 def flush(self):
606 610 if not self.do_full_cache:
607 611 raise ValueError,"You shouldn't have reached the cache flush "\
608 612 "if full caching is not enabled!"
609 613 # delete auto-generated vars from global namespace
610 614
611 615 for n in range(1,self.prompt_count + 1):
612 616 key = '_'+`n`
613 617 try:
614 618 del self.user_ns[key]
615 619 except: pass
616 620 self.user_ns['_oh'].clear()
617 621
618 622 if '_' not in __builtin__.__dict__:
619 623 self.user_ns.update({'_':None,'__':None, '___':None})
620 624 import gc
621 625 gc.collect() # xxx needed?
622 626
@@ -1,875 +1,885 b''
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by starting ipython with the
7 7 flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
8 8 interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import __builtin__
23 23 import commands
24 24 import doctest
25 25 import inspect
26 26 import logging
27 27 import os
28 28 import re
29 29 import sys
30 30 import traceback
31 31 import unittest
32 32
33 33 from inspect import getmodule
34 34 from StringIO import StringIO
35 35
36 36 # We are overriding the default doctest runner, so we need to import a few
37 37 # things from doctest directly
38 38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 39 _unittest_reportflags, DocTestRunner,
40 40 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 41 _exception_traceback,
42 42 linecache)
43 43
44 44 # Third-party modules
45 45 import nose.core
46 46
47 47 from nose.plugins import doctests, Plugin
48 48 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module globals and other constants
52 52
53 53 log = logging.getLogger(__name__)
54 54
55 55 ###########################################################################
56 56 # *** HACK ***
57 57 # We must start our own ipython object and heavily muck with it so that all the
58 58 # modifications IPython makes to system behavior don't send the doctest
59 59 # machinery into a fit. This code should be considered a gross hack, but it
60 60 # gets the job done.
61 61
62 62
63 63 # Hack to modify the %run command so we can sync the user's namespace with the
64 64 # test globals. Once we move over to a clean magic system, this will be done
65 65 # with much less ugliness.
66 66
67 67 class py_file_finder(object):
68 68 def __init__(self,test_filename):
69 69 self.test_filename = test_filename
70 70
71 71 def __call__(self,name):
72 72 from IPython.genutils import get_py_filename
73 73 try:
74 74 return get_py_filename(name)
75 75 except IOError:
76 76 test_dir = os.path.dirname(self.test_filename)
77 77 new_path = os.path.join(test_dir,name)
78 78 return get_py_filename(new_path)
79 79
80 80
81 81 def _run_ns_sync(self,arg_s,runner=None):
82 82 """Modified version of %run that syncs testing namespaces.
83 83
84 84 This is strictly needed for running doctests that call %run.
85 85 """
86 86
87 finder = py_file_finder(_run_ns_sync.test_filename)
87 # When tests call %run directly (not via doctest) these function attributes
88 # are not set
89 try:
90 fname = _run_ns_sync.test_filename
91 except AttributeError:
92 fname = arg_s
93
94 finder = py_file_finder(fname)
88 95 out = _ip.IP.magic_run_ori(arg_s,runner,finder)
96
97 # Simliarly, there is no test_globs when a test is NOT a doctest
98 if hasattr(_run_ns_sync,'test_globs'):
89 99 _run_ns_sync.test_globs.update(_ip.user_ns)
90 100 return out
91 101
92 102
93 103 class ipnsdict(dict):
94 104 """A special subclass of dict for use as an IPython namespace in doctests.
95 105
96 106 This subclass adds a simple checkpointing capability so that when testing
97 107 machinery clears it (we use it as the test execution context), it doesn't
98 108 get completely destroyed.
99 109 """
100 110
101 111 def __init__(self,*a):
102 112 dict.__init__(self,*a)
103 113 self._savedict = {}
104 114
105 115 def clear(self):
106 116 dict.clear(self)
107 117 self.update(self._savedict)
108 118
109 119 def _checkpoint(self):
110 120 self._savedict.clear()
111 121 self._savedict.update(self)
112 122
113 123 def update(self,other):
114 124 self._checkpoint()
115 125 dict.update(self,other)
116 126 # If '_' is in the namespace, python won't set it when executing code,
117 127 # and we have examples that test it. So we ensure that the namespace
118 128 # is always 'clean' of it before it's used for test code execution.
119 129 self.pop('_',None)
120 130
121 131
122 132 def start_ipython():
123 133 """Start a global IPython shell, which we need for IPython-specific syntax.
124 134 """
125 135
126 136 # This function should only ever run once!
127 137 if hasattr(start_ipython,'already_called'):
128 138 return
129 139 start_ipython.already_called = True
130 140
131 141 # Ok, first time we're called, go ahead
132 142 import new
133 143
134 144 import IPython
135 145
136 146 def xsys(cmd):
137 147 """Execute a command and print its output.
138 148
139 149 This is just a convenience function to replace the IPython system call
140 150 with one that is more doctest-friendly.
141 151 """
142 152 cmd = _ip.IP.var_expand(cmd,depth=1)
143 153 sys.stdout.write(commands.getoutput(cmd))
144 154 sys.stdout.flush()
145 155
146 156 # Store certain global objects that IPython modifies
147 157 _displayhook = sys.displayhook
148 158 _excepthook = sys.excepthook
149 159 _main = sys.modules.get('__main__')
150 160
151 161 # Start IPython instance. We customize it to start with minimal frills.
152 162 user_ns,global_ns = IPython.ipapi.make_user_namespaces(ipnsdict(),dict())
153 163 IPython.Shell.IPShell(['--colors=NoColor','--noterm_title'],
154 164 user_ns,global_ns)
155 165
156 166 # Deactivate the various python system hooks added by ipython for
157 167 # interactive convenience so we don't confuse the doctest system
158 168 sys.modules['__main__'] = _main
159 169 sys.displayhook = _displayhook
160 170 sys.excepthook = _excepthook
161 171
162 172 # So that ipython magics and aliases can be doctested (they work by making
163 173 # a call into a global _ip object)
164 174 _ip = IPython.ipapi.get()
165 175 __builtin__._ip = _ip
166 176
167 177 # Modify the IPython system call with one that uses getoutput, so that we
168 178 # can capture subcommands and print them to Python's stdout, otherwise the
169 179 # doctest machinery would miss them.
170 180 _ip.system = xsys
171 181
172 182 # Also patch our %run function in.
173 183 im = new.instancemethod(_run_ns_sync,_ip.IP, _ip.IP.__class__)
174 184 _ip.IP.magic_run_ori = _ip.IP.magic_run
175 185 _ip.IP.magic_run = im
176 186
177 187 # The start call MUST be made here. I'm not sure yet why it doesn't work if
178 188 # it is made later, at plugin initialization time, but in all my tests, that's
179 189 # the case.
180 190 start_ipython()
181 191
182 192 # *** END HACK ***
183 193 ###########################################################################
184 194
185 195 # Classes and functions
186 196
187 197 def is_extension_module(filename):
188 198 """Return whether the given filename is an extension module.
189 199
190 200 This simply checks that the extension is either .so or .pyd.
191 201 """
192 202 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
193 203
194 204
195 205 class DocTestSkip(object):
196 206 """Object wrapper for doctests to be skipped."""
197 207
198 208 ds_skip = """Doctest to skip.
199 209 >>> 1 #doctest: +SKIP
200 210 """
201 211
202 212 def __init__(self,obj):
203 213 self.obj = obj
204 214
205 215 def __getattribute__(self,key):
206 216 if key == '__doc__':
207 217 return DocTestSkip.ds_skip
208 218 else:
209 219 return getattr(object.__getattribute__(self,'obj'),key)
210 220
211 221 # Modified version of the one in the stdlib, that fixes a python bug (doctests
212 222 # not found in extension modules, http://bugs.python.org/issue3158)
213 223 class DocTestFinder(doctest.DocTestFinder):
214 224
215 225 def _from_module(self, module, object):
216 226 """
217 227 Return true if the given object is defined in the given
218 228 module.
219 229 """
220 230 if module is None:
221 231 return True
222 232 elif inspect.isfunction(object):
223 233 return module.__dict__ is object.func_globals
224 234 elif inspect.isbuiltin(object):
225 235 return module.__name__ == object.__module__
226 236 elif inspect.isclass(object):
227 237 return module.__name__ == object.__module__
228 238 elif inspect.ismethod(object):
229 239 # This one may be a bug in cython that fails to correctly set the
230 240 # __module__ attribute of methods, but since the same error is easy
231 241 # to make by extension code writers, having this safety in place
232 242 # isn't such a bad idea
233 243 return module.__name__ == object.im_class.__module__
234 244 elif inspect.getmodule(object) is not None:
235 245 return module is inspect.getmodule(object)
236 246 elif hasattr(object, '__module__'):
237 247 return module.__name__ == object.__module__
238 248 elif isinstance(object, property):
239 249 return True # [XX] no way not be sure.
240 250 else:
241 251 raise ValueError("object must be a class or function")
242 252
243 253 def _find(self, tests, obj, name, module, source_lines, globs, seen):
244 254 """
245 255 Find tests for the given object and any contained objects, and
246 256 add them to `tests`.
247 257 """
248 258
249 259 if hasattr(obj,"skip_doctest"):
250 260 #print 'SKIPPING DOCTEST FOR:',obj # dbg
251 261 obj = DocTestSkip(obj)
252 262
253 263 doctest.DocTestFinder._find(self,tests, obj, name, module,
254 264 source_lines, globs, seen)
255 265
256 266 # Below we re-run pieces of the above method with manual modifications,
257 267 # because the original code is buggy and fails to correctly identify
258 268 # doctests in extension modules.
259 269
260 270 # Local shorthands
261 271 from inspect import isroutine, isclass, ismodule
262 272
263 273 # Look for tests in a module's contained objects.
264 274 if inspect.ismodule(obj) and self._recurse:
265 275 for valname, val in obj.__dict__.items():
266 276 valname1 = '%s.%s' % (name, valname)
267 277 if ( (isroutine(val) or isclass(val))
268 278 and self._from_module(module, val) ):
269 279
270 280 self._find(tests, val, valname1, module, source_lines,
271 281 globs, seen)
272 282
273 283 # Look for tests in a class's contained objects.
274 284 if inspect.isclass(obj) and self._recurse:
275 285 #print 'RECURSE into class:',obj # dbg
276 286 for valname, val in obj.__dict__.items():
277 287 # Special handling for staticmethod/classmethod.
278 288 if isinstance(val, staticmethod):
279 289 val = getattr(obj, valname)
280 290 if isinstance(val, classmethod):
281 291 val = getattr(obj, valname).im_func
282 292
283 293 # Recurse to methods, properties, and nested classes.
284 294 if ((inspect.isfunction(val) or inspect.isclass(val) or
285 295 inspect.ismethod(val) or
286 296 isinstance(val, property)) and
287 297 self._from_module(module, val)):
288 298 valname = '%s.%s' % (name, valname)
289 299 self._find(tests, val, valname, module, source_lines,
290 300 globs, seen)
291 301
292 302
293 303 class IPDoctestOutputChecker(doctest.OutputChecker):
294 304 """Second-chance checker with support for random tests.
295 305
296 306 If the default comparison doesn't pass, this checker looks in the expected
297 307 output string for flags that tell us to ignore the output.
298 308 """
299 309
300 310 random_re = re.compile(r'#\s*random\s+')
301 311
302 312 def check_output(self, want, got, optionflags):
303 313 """Check output, accepting special markers embedded in the output.
304 314
305 315 If the output didn't pass the default validation but the special string
306 316 '#random' is included, we accept it."""
307 317
308 318 # Let the original tester verify first, in case people have valid tests
309 319 # that happen to have a comment saying '#random' embedded in.
310 320 ret = doctest.OutputChecker.check_output(self, want, got,
311 321 optionflags)
312 322 if not ret and self.random_re.search(want):
313 323 #print >> sys.stderr, 'RANDOM OK:',want # dbg
314 324 return True
315 325
316 326 return ret
317 327
318 328
319 329 class DocTestCase(doctests.DocTestCase):
320 330 """Proxy for DocTestCase: provides an address() method that
321 331 returns the correct address for the doctest case. Otherwise
322 332 acts as a proxy to the test case. To provide hints for address(),
323 333 an obj may also be passed -- this will be used as the test object
324 334 for purposes of determining the test address, if it is provided.
325 335 """
326 336
327 337 # Note: this method was taken from numpy's nosetester module.
328 338
329 339 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
330 340 # its constructor that blocks non-default arguments from being passed
331 341 # down into doctest.DocTestCase
332 342
333 343 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
334 344 checker=None, obj=None, result_var='_'):
335 345 self._result_var = result_var
336 346 doctests.DocTestCase.__init__(self, test,
337 347 optionflags=optionflags,
338 348 setUp=setUp, tearDown=tearDown,
339 349 checker=checker)
340 350 # Now we must actually copy the original constructor from the stdlib
341 351 # doctest class, because we can't call it directly and a bug in nose
342 352 # means it never gets passed the right arguments.
343 353
344 354 self._dt_optionflags = optionflags
345 355 self._dt_checker = checker
346 356 self._dt_test = test
347 357 self._dt_setUp = setUp
348 358 self._dt_tearDown = tearDown
349 359
350 360 # XXX - store this runner once in the object!
351 361 runner = IPDocTestRunner(optionflags=optionflags,
352 362 checker=checker, verbose=False)
353 363 self._dt_runner = runner
354 364
355 365
356 366 # Each doctest should remember what directory it was loaded from...
357 367 self._ori_dir = os.getcwd()
358 368
359 369 # Modified runTest from the default stdlib
360 370 def runTest(self):
361 371 test = self._dt_test
362 372 runner = self._dt_runner
363 373
364 374 old = sys.stdout
365 375 new = StringIO()
366 376 optionflags = self._dt_optionflags
367 377
368 378 if not (optionflags & REPORTING_FLAGS):
369 379 # The option flags don't include any reporting flags,
370 380 # so add the default reporting flags
371 381 optionflags |= _unittest_reportflags
372 382
373 383 try:
374 384 # Save our current directory and switch out to the one where the
375 385 # test was originally created, in case another doctest did a
376 386 # directory change. We'll restore this in the finally clause.
377 387 curdir = os.getcwd()
378 388 os.chdir(self._ori_dir)
379 389
380 390 runner.DIVIDER = "-"*70
381 391 failures, tries = runner.run(test,out=new.write,
382 392 clear_globs=False)
383 393 finally:
384 394 sys.stdout = old
385 395 os.chdir(curdir)
386 396
387 397 if failures:
388 398 raise self.failureException(self.format_failure(new.getvalue()))
389 399
390 400 def setUp(self):
391 401 """Modified test setup that syncs with ipython namespace"""
392 402
393 403 if isinstance(self._dt_test.examples[0],IPExample):
394 404 # for IPython examples *only*, we swap the globals with the ipython
395 405 # namespace, after updating it with the globals (which doctest
396 406 # fills with the necessary info from the module being tested).
397 407 _ip.IP.user_ns.update(self._dt_test.globs)
398 408 self._dt_test.globs = _ip.IP.user_ns
399 409
400 410 doctests.DocTestCase.setUp(self)
401 411
402 412
403 413 # A simple subclassing of the original with a different class name, so we can
404 414 # distinguish and treat differently IPython examples from pure python ones.
405 415 class IPExample(doctest.Example): pass
406 416
407 417
408 418 class IPExternalExample(doctest.Example):
409 419 """Doctest examples to be run in an external process."""
410 420
411 421 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
412 422 options=None):
413 423 # Parent constructor
414 424 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
415 425
416 426 # An EXTRA newline is needed to prevent pexpect hangs
417 427 self.source += '\n'
418 428
419 429
420 430 class IPDocTestParser(doctest.DocTestParser):
421 431 """
422 432 A class used to parse strings containing doctest examples.
423 433
424 434 Note: This is a version modified to properly recognize IPython input and
425 435 convert any IPython examples into valid Python ones.
426 436 """
427 437 # This regular expression is used to find doctest examples in a
428 438 # string. It defines three groups: `source` is the source code
429 439 # (including leading indentation and prompts); `indent` is the
430 440 # indentation of the first (PS1) line of the source code; and
431 441 # `want` is the expected output (including leading indentation).
432 442
433 443 # Classic Python prompts or default IPython ones
434 444 _PS1_PY = r'>>>'
435 445 _PS2_PY = r'\.\.\.'
436 446
437 447 _PS1_IP = r'In\ \[\d+\]:'
438 448 _PS2_IP = r'\ \ \ \.\.\.+:'
439 449
440 450 _RE_TPL = r'''
441 451 # Source consists of a PS1 line followed by zero or more PS2 lines.
442 452 (?P<source>
443 453 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
444 454 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
445 455 \n? # a newline
446 456 # Want consists of any non-blank lines that do not start with PS1.
447 457 (?P<want> (?:(?![ ]*$) # Not a blank line
448 458 (?![ ]*%s) # Not a line starting with PS1
449 459 (?![ ]*%s) # Not a line starting with PS2
450 460 .*$\n? # But any other line
451 461 )*)
452 462 '''
453 463
454 464 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
455 465 re.MULTILINE | re.VERBOSE)
456 466
457 467 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
458 468 re.MULTILINE | re.VERBOSE)
459 469
460 470 # Mark a test as being fully random. In this case, we simply append the
461 471 # random marker ('#random') to each individual example's output. This way
462 472 # we don't need to modify any other code.
463 473 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
464 474
465 475 # Mark tests to be executed in an external process - currently unsupported.
466 476 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
467 477
468 478 def ip2py(self,source):
469 479 """Convert input IPython source into valid Python."""
470 480 out = []
471 481 newline = out.append
472 482 #print 'IPSRC:\n',source,'\n###' # dbg
473 483 # The input source must be first stripped of all bracketing whitespace
474 484 # and turned into lines, so it looks to the parser like regular user
475 485 # input
476 486 for lnum,line in enumerate(source.strip().splitlines()):
477 487 newline(_ip.IP.prefilter(line,lnum>0))
478 488 newline('') # ensure a closing newline, needed by doctest
479 489 #print "PYSRC:", '\n'.join(out) # dbg
480 490 return '\n'.join(out)
481 491
482 492 def parse(self, string, name='<string>'):
483 493 """
484 494 Divide the given string into examples and intervening text,
485 495 and return them as a list of alternating Examples and strings.
486 496 Line numbers for the Examples are 0-based. The optional
487 497 argument `name` is a name identifying this string, and is only
488 498 used for error messages.
489 499 """
490 500
491 501 #print 'Parse string:\n',string # dbg
492 502
493 503 string = string.expandtabs()
494 504 # If all lines begin with the same indentation, then strip it.
495 505 min_indent = self._min_indent(string)
496 506 if min_indent > 0:
497 507 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
498 508
499 509 output = []
500 510 charno, lineno = 0, 0
501 511
502 512 # We make 'all random' tests by adding the '# random' mark to every
503 513 # block of output in the test.
504 514 if self._RANDOM_TEST.search(string):
505 515 random_marker = '\n# random'
506 516 else:
507 517 random_marker = ''
508 518
509 519 # Whether to convert the input from ipython to python syntax
510 520 ip2py = False
511 521 # Find all doctest examples in the string. First, try them as Python
512 522 # examples, then as IPython ones
513 523 terms = list(self._EXAMPLE_RE_PY.finditer(string))
514 524 if terms:
515 525 # Normal Python example
516 526 #print '-'*70 # dbg
517 527 #print 'PyExample, Source:\n',string # dbg
518 528 #print '-'*70 # dbg
519 529 Example = doctest.Example
520 530 else:
521 531 # It's an ipython example. Note that IPExamples are run
522 532 # in-process, so their syntax must be turned into valid python.
523 533 # IPExternalExamples are run out-of-process (via pexpect) so they
524 534 # don't need any filtering (a real ipython will be executing them).
525 535 terms = list(self._EXAMPLE_RE_IP.finditer(string))
526 536 if self._EXTERNAL_IP.search(string):
527 537 #print '-'*70 # dbg
528 538 #print 'IPExternalExample, Source:\n',string # dbg
529 539 #print '-'*70 # dbg
530 540 Example = IPExternalExample
531 541 else:
532 542 #print '-'*70 # dbg
533 543 #print 'IPExample, Source:\n',string # dbg
534 544 #print '-'*70 # dbg
535 545 Example = IPExample
536 546 ip2py = True
537 547
538 548 for m in terms:
539 549 # Add the pre-example text to `output`.
540 550 output.append(string[charno:m.start()])
541 551 # Update lineno (lines before this example)
542 552 lineno += string.count('\n', charno, m.start())
543 553 # Extract info from the regexp match.
544 554 (source, options, want, exc_msg) = \
545 555 self._parse_example(m, name, lineno,ip2py)
546 556
547 557 # Append the random-output marker (it defaults to empty in most
548 558 # cases, it's only non-empty for 'all-random' tests):
549 559 want += random_marker
550 560
551 561 if Example is IPExternalExample:
552 562 options[doctest.NORMALIZE_WHITESPACE] = True
553 563 want += '\n'
554 564
555 565 # Create an Example, and add it to the list.
556 566 if not self._IS_BLANK_OR_COMMENT(source):
557 567 output.append(Example(source, want, exc_msg,
558 568 lineno=lineno,
559 569 indent=min_indent+len(m.group('indent')),
560 570 options=options))
561 571 # Update lineno (lines inside this example)
562 572 lineno += string.count('\n', m.start(), m.end())
563 573 # Update charno.
564 574 charno = m.end()
565 575 # Add any remaining post-example text to `output`.
566 576 output.append(string[charno:])
567 577 return output
568 578
569 579 def _parse_example(self, m, name, lineno,ip2py=False):
570 580 """
571 581 Given a regular expression match from `_EXAMPLE_RE` (`m`),
572 582 return a pair `(source, want)`, where `source` is the matched
573 583 example's source code (with prompts and indentation stripped);
574 584 and `want` is the example's expected output (with indentation
575 585 stripped).
576 586
577 587 `name` is the string's name, and `lineno` is the line number
578 588 where the example starts; both are used for error messages.
579 589
580 590 Optional:
581 591 `ip2py`: if true, filter the input via IPython to convert the syntax
582 592 into valid python.
583 593 """
584 594
585 595 # Get the example's indentation level.
586 596 indent = len(m.group('indent'))
587 597
588 598 # Divide source into lines; check that they're properly
589 599 # indented; and then strip their indentation & prompts.
590 600 source_lines = m.group('source').split('\n')
591 601
592 602 # We're using variable-length input prompts
593 603 ps1 = m.group('ps1')
594 604 ps2 = m.group('ps2')
595 605 ps1_len = len(ps1)
596 606
597 607 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
598 608 if ps2:
599 609 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
600 610
601 611 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
602 612
603 613 if ip2py:
604 614 # Convert source input from IPython into valid Python syntax
605 615 source = self.ip2py(source)
606 616
607 617 # Divide want into lines; check that it's properly indented; and
608 618 # then strip the indentation. Spaces before the last newline should
609 619 # be preserved, so plain rstrip() isn't good enough.
610 620 want = m.group('want')
611 621 want_lines = want.split('\n')
612 622 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
613 623 del want_lines[-1] # forget final newline & spaces after it
614 624 self._check_prefix(want_lines, ' '*indent, name,
615 625 lineno + len(source_lines))
616 626
617 627 # Remove ipython output prompt that might be present in the first line
618 628 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
619 629
620 630 want = '\n'.join([wl[indent:] for wl in want_lines])
621 631
622 632 # If `want` contains a traceback message, then extract it.
623 633 m = self._EXCEPTION_RE.match(want)
624 634 if m:
625 635 exc_msg = m.group('msg')
626 636 else:
627 637 exc_msg = None
628 638
629 639 # Extract options from the source.
630 640 options = self._find_options(source, name, lineno)
631 641
632 642 return source, options, want, exc_msg
633 643
634 644 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
635 645 """
636 646 Given the lines of a source string (including prompts and
637 647 leading indentation), check to make sure that every prompt is
638 648 followed by a space character. If any line is not followed by
639 649 a space character, then raise ValueError.
640 650
641 651 Note: IPython-modified version which takes the input prompt length as a
642 652 parameter, so that prompts of variable length can be dealt with.
643 653 """
644 654 space_idx = indent+ps1_len
645 655 min_len = space_idx+1
646 656 for i, line in enumerate(lines):
647 657 if len(line) >= min_len and line[space_idx] != ' ':
648 658 raise ValueError('line %r of the docstring for %s '
649 659 'lacks blank after %s: %r' %
650 660 (lineno+i+1, name,
651 661 line[indent:space_idx], line))
652 662
653 663
654 664 SKIP = doctest.register_optionflag('SKIP')
655 665
656 666
657 667 class IPDocTestRunner(doctest.DocTestRunner,object):
658 668 """Test runner that synchronizes the IPython namespace with test globals.
659 669 """
660 670
661 671 def run(self, test, compileflags=None, out=None, clear_globs=True):
662 672
663 673 # Hack: ipython needs access to the execution context of the example,
664 674 # so that it can propagate user variables loaded by %run into
665 675 # test.globs. We put them here into our modified %run as a function
666 676 # attribute. Our new %run will then only make the namespace update
667 677 # when called (rather than unconconditionally updating test.globs here
668 678 # for all examples, most of which won't be calling %run anyway).
669 679 _run_ns_sync.test_globs = test.globs
670 680 _run_ns_sync.test_filename = test.filename
671 681
672 682 return super(IPDocTestRunner,self).run(test,
673 683 compileflags,out,clear_globs)
674 684
675 685
676 686 class DocFileCase(doctest.DocFileCase):
677 687 """Overrides to provide filename
678 688 """
679 689 def address(self):
680 690 return (self._dt_test.filename, None, None)
681 691
682 692
683 693 class ExtensionDoctest(doctests.Doctest):
684 694 """Nose Plugin that supports doctests in extension modules.
685 695 """
686 696 name = 'extdoctest' # call nosetests with --with-extdoctest
687 697 enabled = True
688 698
689 699 def __init__(self,exclude_patterns=None):
690 700 """Create a new ExtensionDoctest plugin.
691 701
692 702 Parameters
693 703 ----------
694 704
695 705 exclude_patterns : sequence of strings, optional
696 706 These patterns are compiled as regular expressions, subsequently used
697 707 to exclude any filename which matches them from inclusion in the test
698 708 suite (using pattern.search(), NOT pattern.match() ).
699 709 """
700 710
701 711 if exclude_patterns is None:
702 712 exclude_patterns = []
703 713 self.exclude_patterns = map(re.compile,exclude_patterns)
704 714 doctests.Doctest.__init__(self)
705 715
706 716 def options(self, parser, env=os.environ):
707 717 Plugin.options(self, parser, env)
708 718 parser.add_option('--doctest-tests', action='store_true',
709 719 dest='doctest_tests',
710 720 default=env.get('NOSE_DOCTEST_TESTS',True),
711 721 help="Also look for doctests in test modules. "
712 722 "Note that classes, methods and functions should "
713 723 "have either doctests or non-doctest tests, "
714 724 "not both. [NOSE_DOCTEST_TESTS]")
715 725 parser.add_option('--doctest-extension', action="append",
716 726 dest="doctestExtension",
717 727 help="Also look for doctests in files with "
718 728 "this extension [NOSE_DOCTEST_EXTENSION]")
719 729 # Set the default as a list, if given in env; otherwise
720 730 # an additional value set on the command line will cause
721 731 # an error.
722 732 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
723 733 if env_setting is not None:
724 734 parser.set_defaults(doctestExtension=tolist(env_setting))
725 735
726 736
727 737 def configure(self, options, config):
728 738 Plugin.configure(self, options, config)
729 739 self.doctest_tests = options.doctest_tests
730 740 self.extension = tolist(options.doctestExtension)
731 741
732 742 self.parser = doctest.DocTestParser()
733 743 self.finder = DocTestFinder()
734 744 self.checker = IPDoctestOutputChecker()
735 745 self.globs = None
736 746 self.extraglobs = None
737 747
738 748
739 749 def loadTestsFromExtensionModule(self,filename):
740 750 bpath,mod = os.path.split(filename)
741 751 modname = os.path.splitext(mod)[0]
742 752 try:
743 753 sys.path.append(bpath)
744 754 module = __import__(modname)
745 755 tests = list(self.loadTestsFromModule(module))
746 756 finally:
747 757 sys.path.pop()
748 758 return tests
749 759
750 760 # NOTE: the method below is almost a copy of the original one in nose, with
751 761 # a few modifications to control output checking.
752 762
753 763 def loadTestsFromModule(self, module):
754 764 #print '*** ipdoctest - lTM',module # dbg
755 765
756 766 if not self.matches(module.__name__):
757 767 log.debug("Doctest doesn't want module %s", module)
758 768 return
759 769
760 770 tests = self.finder.find(module,globs=self.globs,
761 771 extraglobs=self.extraglobs)
762 772 if not tests:
763 773 return
764 774
765 775 # always use whitespace and ellipsis options
766 776 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
767 777
768 778 tests.sort()
769 779 module_file = module.__file__
770 780 if module_file[-4:] in ('.pyc', '.pyo'):
771 781 module_file = module_file[:-1]
772 782 for test in tests:
773 783 if not test.examples:
774 784 continue
775 785 if not test.filename:
776 786 test.filename = module_file
777 787
778 788 yield DocTestCase(test,
779 789 optionflags=optionflags,
780 790 checker=self.checker)
781 791
782 792
783 793 def loadTestsFromFile(self, filename):
784 794 if is_extension_module(filename):
785 795 for t in self.loadTestsFromExtensionModule(filename):
786 796 yield t
787 797 else:
788 798 if self.extension and anyp(filename.endswith, self.extension):
789 799 name = os.path.basename(filename)
790 800 dh = open(filename)
791 801 try:
792 802 doc = dh.read()
793 803 finally:
794 804 dh.close()
795 805 test = self.parser.get_doctest(
796 806 doc, globs={'__file__': filename}, name=name,
797 807 filename=filename, lineno=0)
798 808 if test.examples:
799 809 #print 'FileCase:',test.examples # dbg
800 810 yield DocFileCase(test)
801 811 else:
802 812 yield False # no tests to load
803 813
804 814 def wantFile(self,filename):
805 815 """Return whether the given filename should be scanned for tests.
806 816
807 817 Modified version that accepts extension modules as valid containers for
808 818 doctests.
809 819 """
810 820 #print '*** ipdoctest- wantFile:',filename # dbg
811 821
812 822 for pat in self.exclude_patterns:
813 823 if pat.search(filename):
814 824 #print '###>>> SKIP:',filename # dbg
815 825 return False
816 826
817 827 if is_extension_module(filename):
818 828 return True
819 829 else:
820 830 return doctests.Doctest.wantFile(self,filename)
821 831
822 832
823 833 class IPythonDoctest(ExtensionDoctest):
824 834 """Nose Plugin that supports doctests in extension modules.
825 835 """
826 836 name = 'ipdoctest' # call nosetests with --with-ipdoctest
827 837 enabled = True
828 838
829 839 def makeTest(self, obj, parent):
830 840 """Look for doctests in the given object, which will be a
831 841 function, method or class.
832 842 """
833 843 # always use whitespace and ellipsis options
834 844 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
835 845
836 846 doctests = self.finder.find(obj, module=getmodule(parent))
837 847 if doctests:
838 848 for test in doctests:
839 849 if len(test.examples) == 0:
840 850 continue
841 851
842 852 yield DocTestCase(test, obj=obj,
843 853 optionflags=optionflags,
844 854 checker=self.checker)
845 855
846 856 def options(self, parser, env=os.environ):
847 857 Plugin.options(self, parser, env)
848 858 parser.add_option('--ipdoctest-tests', action='store_true',
849 859 dest='ipdoctest_tests',
850 860 default=env.get('NOSE_IPDOCTEST_TESTS',True),
851 861 help="Also look for doctests in test modules. "
852 862 "Note that classes, methods and functions should "
853 863 "have either doctests or non-doctest tests, "
854 864 "not both. [NOSE_IPDOCTEST_TESTS]")
855 865 parser.add_option('--ipdoctest-extension', action="append",
856 866 dest="ipdoctest_extension",
857 867 help="Also look for doctests in files with "
858 868 "this extension [NOSE_IPDOCTEST_EXTENSION]")
859 869 # Set the default as a list, if given in env; otherwise
860 870 # an additional value set on the command line will cause
861 871 # an error.
862 872 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
863 873 if env_setting is not None:
864 874 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
865 875
866 876 def configure(self, options, config):
867 877 Plugin.configure(self, options, config)
868 878 self.doctest_tests = options.ipdoctest_tests
869 879 self.extension = tolist(options.ipdoctest_extension)
870 880
871 881 self.parser = IPDocTestParser()
872 882 self.finder = DocTestFinder(parser=self.parser)
873 883 self.checker = IPDoctestOutputChecker()
874 884 self.globs = None
875 885 self.extraglobs = None
@@ -1,151 +1,203 b''
1 1 """Tests for various magic functions.
2 2
3 3 Needs to be run by nose (to make ipython session available).
4 4 """
5 5
6 6 # Standard library imports
7 7 import os
8 8 import sys
9 9
10 10 # Third-party imports
11 11 import nose.tools as nt
12 12
13 13 # From our own code
14 14 from IPython.testing import decorators as dec
15
16 15 #-----------------------------------------------------------------------------
17 16 # Test functions begin
18 17
19 18 def test_rehashx():
20 19 # clear up everything
21 20 _ip.IP.alias_table.clear()
22 21 del _ip.db['syscmdlist']
23 22
24 23 _ip.magic('rehashx')
25 24 # Practically ALL ipython development systems will have more than 10 aliases
26 25
27 26 assert len(_ip.IP.alias_table) > 10
28 27 for key, val in _ip.IP.alias_table.items():
29 28 # we must strip dots from alias names
30 29 assert '.' not in key
31 30
32 31 # rehashx must fill up syscmdlist
33 32 scoms = _ip.db['syscmdlist']
34 33 assert len(scoms) > 10
35 34
36 35
37 36 def doctest_run_ns():
38 37 """Classes declared %run scripts must be instantiable afterwards.
39 38
40 39 In [11]: run tclass foo
41 40
42 41 In [12]: isinstance(f(),foo)
43 42 Out[12]: True
44 43 """
45 44
46 45
47 46 def doctest_run_ns2():
48 47 """Classes declared %run scripts must be instantiable afterwards.
49 48
50 49 In [4]: run tclass C-first_pass
51 50
52 51 In [5]: run tclass C-second_pass
53 52 tclass.py: deleting object: C-first_pass
54 53 """
55 54
56 55
57 56 def doctest_hist_f():
58 57 """Test %hist -f with temporary filename.
59 58
60 59 In [9]: import tempfile
61 60
62 61 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
63 62
64 63 In [11]: %history -n -f $tfile 3
65 64 """
66 65
66 def doctest_run_builtins():
67 """Check that %run doesn't damage __builtins__ via a doctest.
68
69 This is similar to the test_run_builtins, but I want *both* forms of the
70 test to catch any possible glitches in our testing machinery, since that
71 modifies %run somewhat. So for this, we have both a normal test (below)
72 and a doctest (this one).
73
74 In [1]: import tempfile
75
76 In [2]: bid1 = id(__builtins__)
77
78 In [3]: f = tempfile.NamedTemporaryFile()
79
80 In [4]: f.write('pass\\n')
81
82 In [5]: f.flush()
83
84 In [6]: print 'B1:',type(__builtins__)
85 B1: <type 'module'>
86
87 In [7]: %run $f.name
88
89 In [8]: bid2 = id(__builtins__)
90
91 In [9]: print 'B2:',type(__builtins__)
92 B2: <type 'module'>
93
94 In [10]: bid1 == bid2
95 Out[10]: True
96 """
97
98 def test_run_builtins():
99 """Check that %run doesn't damage __builtins__ """
100 import sys
101 import tempfile
102 import types
103
104 # Make an empty file and put 'pass' in it
105 f = tempfile.NamedTemporaryFile()
106 f.write('pass\n')
107 f.flush()
108
109 # Our first test is that the id of __builtins__ is not modified by %run
110 bid1 = id(__builtins__)
111 _ip.magic('run %s' % f.name)
112 bid2 = id(__builtins__)
113 yield nt.assert_equals,bid1,bid2
114 # However, the above could pass if __builtins__ was already modified to be
115 # a dict (it should be a module) by a previous use of %run. So we also
116 # check explicitly that it really is a module:
117 yield nt.assert_equals,type(__builtins__),type(sys)
118
67 119
68 120 def doctest_hist_r():
69 121 """Test %hist -r
70 122
71 123 XXX - This test is not recording the output correctly. Not sure why...
72 124
73 125 In [6]: x=1
74 126
75 127 In [7]: hist -n -r 2
76 128 x=1 # random
77 129 hist -n -r 2 # random
78 130 """
79 131
80 132
81 133 def test_obj_del():
82 134 """Test that object's __del__ methods are called on exit."""
83 135 test_dir = os.path.dirname(__file__)
84 136 del_file = os.path.join(test_dir,'obj_del.py')
85 137 out = _ip.IP.getoutput('ipython %s' % del_file)
86 138 nt.assert_equals(out,'obj_del.py: object A deleted')
87 139
88 140
89 141 def test_shist():
90 142 # Simple tests of ShadowHist class - test generator.
91 143 import os, shutil, tempfile
92 144
93 145 from IPython.Extensions import pickleshare
94 146 from IPython.history import ShadowHist
95 147
96 148 tfile = tempfile.mktemp('','tmp-ipython-')
97 149
98 150 db = pickleshare.PickleShareDB(tfile)
99 151 s = ShadowHist(db)
100 152 s.add('hello')
101 153 s.add('world')
102 154 s.add('hello')
103 155 s.add('hello')
104 156 s.add('karhu')
105 157
106 158 yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
107 159
108 160 yield nt.assert_equal,s.get(2),'world'
109 161
110 162 shutil.rmtree(tfile)
111 163
112 164 @dec.skipif_not_numpy
113 165 def test_numpy_clear_array_undec():
114 166 _ip.ex('import numpy as np')
115 167 _ip.ex('a = np.empty(2)')
116 168
117 169 yield nt.assert_true,'a' in _ip.user_ns
118 170 _ip.magic('clear array')
119 171 yield nt.assert_false,'a' in _ip.user_ns
120 172
121 173
122 174 @dec.skip()
123 175 def test_fail_dec(*a,**k):
124 176 yield nt.assert_true, False
125 177
126 178 @dec.skip('This one shouldn not run')
127 179 def test_fail_dec2(*a,**k):
128 180 yield nt.assert_true, False
129 181
130 182 @dec.skipknownfailure
131 183 def test_fail_dec3(*a,**k):
132 184 yield nt.assert_true, False
133 185
134 186
135 187 def doctest_refbug():
136 188 """Very nasty problem with references held by multiple runs of a script.
137 189 See: https://bugs.launchpad.net/ipython/+bug/269966
138 190
139 191 In [1]: _ip.IP.clear_main_mod_cache()
140 192
141 193 In [2]: run refbug
142 194
143 195 In [3]: call_f()
144 196 lowercased: hello
145 197
146 198 In [4]: run refbug
147 199
148 200 In [5]: call_f()
149 201 lowercased: hello
150 202 lowercased: hello
151 203 """
General Comments 0
You need to be logged in to leave comments. Login now