##// END OF EJS Templates
Use user_ns as global namespace if it is passed without user_module, and add test for pickling interactively defined objects....
Thomas Kluyver -
Show More
@@ -1,2648 +1,2666 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 from __future__ import with_statement
18 18 from __future__ import absolute_import
19 19
20 20 import __builtin__ as builtin_mod
21 21 import __future__
22 22 import abc
23 23 import ast
24 24 import atexit
25 25 import codeop
26 26 import inspect
27 27 import os
28 28 import re
29 29 import sys
30 30 import tempfile
31 31 import types
32 32 try:
33 33 from contextlib import nested
34 34 except:
35 35 from IPython.utils.nested_context import nested
36 36
37 37 from IPython.config.configurable import SingletonConfigurable
38 38 from IPython.core import debugger, oinspect
39 39 from IPython.core import history as ipcorehist
40 40 from IPython.core import page
41 41 from IPython.core import prefilter
42 42 from IPython.core import shadowns
43 43 from IPython.core import ultratb
44 44 from IPython.core.alias import AliasManager, AliasError
45 45 from IPython.core.autocall import ExitAutocall
46 46 from IPython.core.builtin_trap import BuiltinTrap
47 47 from IPython.core.compilerop import CachingCompiler
48 48 from IPython.core.display_trap import DisplayTrap
49 49 from IPython.core.displayhook import DisplayHook
50 50 from IPython.core.displaypub import DisplayPublisher
51 51 from IPython.core.error import TryNext, UsageError
52 52 from IPython.core.extensions import ExtensionManager
53 53 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
54 54 from IPython.core.formatters import DisplayFormatter
55 55 from IPython.core.history import HistoryManager
56 56 from IPython.core.inputsplitter import IPythonInputSplitter
57 57 from IPython.core.logger import Logger
58 58 from IPython.core.macro import Macro
59 59 from IPython.core.magic import Magic
60 60 from IPython.core.payload import PayloadManager
61 61 from IPython.core.plugin import PluginManager
62 62 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
63 63 from IPython.core.profiledir import ProfileDir
64 64 from IPython.external.Itpl import ItplNS
65 65 from IPython.utils import PyColorize
66 66 from IPython.utils import io
67 67 from IPython.utils import py3compat
68 68 from IPython.utils.doctestreload import doctest_reload
69 69 from IPython.utils.io import ask_yes_no, rprint
70 70 from IPython.utils.ipstruct import Struct
71 71 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
72 72 from IPython.utils.pickleshare import PickleShareDB
73 73 from IPython.utils.process import system, getoutput
74 74 from IPython.utils.strdispatch import StrDispatch
75 75 from IPython.utils.syspathcontext import prepended_to_syspath
76 76 from IPython.utils.text import (num_ini_spaces, format_screen, LSString, SList,
77 77 DollarFormatter)
78 78 from IPython.utils.traitlets import (Integer, CBool, CaselessStrEnum, Enum,
79 79 List, Unicode, Instance, Type)
80 80 from IPython.utils.warn import warn, error, fatal
81 81 import IPython.core.hooks
82 82
83 83 #-----------------------------------------------------------------------------
84 84 # Globals
85 85 #-----------------------------------------------------------------------------
86 86
87 87 # compiled regexps for autoindent management
88 88 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
89 89
90 90 #-----------------------------------------------------------------------------
91 91 # Utilities
92 92 #-----------------------------------------------------------------------------
93 93
94 94 def softspace(file, newvalue):
95 95 """Copied from code.py, to remove the dependency"""
96 96
97 97 oldvalue = 0
98 98 try:
99 99 oldvalue = file.softspace
100 100 except AttributeError:
101 101 pass
102 102 try:
103 103 file.softspace = newvalue
104 104 except (AttributeError, TypeError):
105 105 # "attribute-less object" or "read-only attributes"
106 106 pass
107 107 return oldvalue
108 108
109 109
110 110 def no_op(*a, **kw): pass
111 111
112 112 class NoOpContext(object):
113 113 def __enter__(self): pass
114 114 def __exit__(self, type, value, traceback): pass
115 115 no_op_context = NoOpContext()
116 116
117 117 class SpaceInInput(Exception): pass
118 118
119 119 class Bunch: pass
120 120
121 121
122 122 def get_default_colors():
123 123 if sys.platform=='darwin':
124 124 return "LightBG"
125 125 elif os.name=='nt':
126 126 return 'Linux'
127 127 else:
128 128 return 'Linux'
129 129
130 130
131 131 class SeparateUnicode(Unicode):
132 132 """A Unicode subclass to validate separate_in, separate_out, etc.
133 133
134 134 This is a Unicode based trait that converts '0'->'' and '\\n'->'\n'.
135 135 """
136 136
137 137 def validate(self, obj, value):
138 138 if value == '0': value = ''
139 139 value = value.replace('\\n','\n')
140 140 return super(SeparateUnicode, self).validate(obj, value)
141 141
142 142
143 143 class ReadlineNoRecord(object):
144 144 """Context manager to execute some code, then reload readline history
145 145 so that interactive input to the code doesn't appear when pressing up."""
146 146 def __init__(self, shell):
147 147 self.shell = shell
148 148 self._nested_level = 0
149 149
150 150 def __enter__(self):
151 151 if self._nested_level == 0:
152 152 try:
153 153 self.orig_length = self.current_length()
154 154 self.readline_tail = self.get_readline_tail()
155 155 except (AttributeError, IndexError): # Can fail with pyreadline
156 156 self.orig_length, self.readline_tail = 999999, []
157 157 self._nested_level += 1
158 158
159 159 def __exit__(self, type, value, traceback):
160 160 self._nested_level -= 1
161 161 if self._nested_level == 0:
162 162 # Try clipping the end if it's got longer
163 163 try:
164 164 e = self.current_length() - self.orig_length
165 165 if e > 0:
166 166 for _ in range(e):
167 167 self.shell.readline.remove_history_item(self.orig_length)
168 168
169 169 # If it still doesn't match, just reload readline history.
170 170 if self.current_length() != self.orig_length \
171 171 or self.get_readline_tail() != self.readline_tail:
172 172 self.shell.refill_readline_hist()
173 173 except (AttributeError, IndexError):
174 174 pass
175 175 # Returning False will cause exceptions to propagate
176 176 return False
177 177
178 178 def current_length(self):
179 179 return self.shell.readline.get_current_history_length()
180 180
181 181 def get_readline_tail(self, n=10):
182 182 """Get the last n items in readline history."""
183 183 end = self.shell.readline.get_current_history_length() + 1
184 184 start = max(end-n, 1)
185 185 ghi = self.shell.readline.get_history_item
186 186 return [ghi(x) for x in range(start, end)]
187 187
188 188
189 189 _autocall_help = """
190 190 Make IPython automatically call any callable object even if
191 191 you didn't type explicit parentheses. For example, 'str 43' becomes 'str(43)'
192 192 automatically. The value can be '0' to disable the feature, '1' for 'smart'
193 193 autocall, where it is not applied if there are no more arguments on the line,
194 194 and '2' for 'full' autocall, where all callable objects are automatically
195 195 called (even if no arguments are present). The default is '1'.
196 196 """
197 197
198 198 #-----------------------------------------------------------------------------
199 199 # Main IPython class
200 200 #-----------------------------------------------------------------------------
201 201
202 202 class InteractiveShell(SingletonConfigurable, Magic):
203 203 """An enhanced, interactive shell for Python."""
204 204
205 205 _instance = None
206 206
207 207 autocall = Enum((0,1,2), default_value=1, config=True, help=
208 208 """
209 209 Make IPython automatically call any callable object even if you didn't
210 210 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
211 211 automatically. The value can be '0' to disable the feature, '1' for
212 212 'smart' autocall, where it is not applied if there are no more
213 213 arguments on the line, and '2' for 'full' autocall, where all callable
214 214 objects are automatically called (even if no arguments are present).
215 215 The default is '1'.
216 216 """
217 217 )
218 218 # TODO: remove all autoindent logic and put into frontends.
219 219 # We can't do this yet because even runlines uses the autoindent.
220 220 autoindent = CBool(True, config=True, help=
221 221 """
222 222 Autoindent IPython code entered interactively.
223 223 """
224 224 )
225 225 automagic = CBool(True, config=True, help=
226 226 """
227 227 Enable magic commands to be called without the leading %.
228 228 """
229 229 )
230 230 cache_size = Integer(1000, config=True, help=
231 231 """
232 232 Set the size of the output cache. The default is 1000, you can
233 233 change it permanently in your config file. Setting it to 0 completely
234 234 disables the caching system, and the minimum value accepted is 20 (if
235 235 you provide a value less than 20, it is reset to 0 and a warning is
236 236 issued). This limit is defined because otherwise you'll spend more
237 237 time re-flushing a too small cache than working
238 238 """
239 239 )
240 240 color_info = CBool(True, config=True, help=
241 241 """
242 242 Use colors for displaying information about objects. Because this
243 243 information is passed through a pager (like 'less'), and some pagers
244 244 get confused with color codes, this capability can be turned off.
245 245 """
246 246 )
247 247 colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
248 248 default_value=get_default_colors(), config=True,
249 249 help="Set the color scheme (NoColor, Linux, or LightBG)."
250 250 )
251 251 colors_force = CBool(False, help=
252 252 """
253 253 Force use of ANSI color codes, regardless of OS and readline
254 254 availability.
255 255 """
256 256 # FIXME: This is essentially a hack to allow ZMQShell to show colors
257 257 # without readline on Win32. When the ZMQ formatting system is
258 258 # refactored, this should be removed.
259 259 )
260 260 debug = CBool(False, config=True)
261 261 deep_reload = CBool(False, config=True, help=
262 262 """
263 263 Enable deep (recursive) reloading by default. IPython can use the
264 264 deep_reload module which reloads changes in modules recursively (it
265 265 replaces the reload() function, so you don't need to change anything to
266 266 use it). deep_reload() forces a full reload of modules whose code may
267 267 have changed, which the default reload() function does not. When
268 268 deep_reload is off, IPython will use the normal reload(), but
269 269 deep_reload will still be available as dreload().
270 270 """
271 271 )
272 272 display_formatter = Instance(DisplayFormatter)
273 273 displayhook_class = Type(DisplayHook)
274 274 display_pub_class = Type(DisplayPublisher)
275 275
276 276 exit_now = CBool(False)
277 277 exiter = Instance(ExitAutocall)
278 278 def _exiter_default(self):
279 279 return ExitAutocall(self)
280 280 # Monotonically increasing execution counter
281 281 execution_count = Integer(1)
282 282 filename = Unicode("<ipython console>")
283 283 ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
284 284
285 285 # Input splitter, to split entire cells of input into either individual
286 286 # interactive statements or whole blocks.
287 287 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
288 288 (), {})
289 289 logstart = CBool(False, config=True, help=
290 290 """
291 291 Start logging to the default log file.
292 292 """
293 293 )
294 294 logfile = Unicode('', config=True, help=
295 295 """
296 296 The name of the logfile to use.
297 297 """
298 298 )
299 299 logappend = Unicode('', config=True, help=
300 300 """
301 301 Start logging to the given file in append mode.
302 302 """
303 303 )
304 304 object_info_string_level = Enum((0,1,2), default_value=0,
305 305 config=True)
306 306 pdb = CBool(False, config=True, help=
307 307 """
308 308 Automatically call the pdb debugger after every exception.
309 309 """
310 310 )
311 311 multiline_history = CBool(sys.platform != 'win32', config=True,
312 312 help="Save multi-line entries as one entry in readline history"
313 313 )
314 314
315 315 prompt_in1 = Unicode('In [\\#]: ', config=True)
316 316 prompt_in2 = Unicode(' .\\D.: ', config=True)
317 317 prompt_out = Unicode('Out[\\#]: ', config=True)
318 318 prompts_pad_left = CBool(True, config=True)
319 319 quiet = CBool(False, config=True)
320 320
321 321 history_length = Integer(10000, config=True)
322 322
323 323 # The readline stuff will eventually be moved to the terminal subclass
324 324 # but for now, we can't do that as readline is welded in everywhere.
325 325 readline_use = CBool(True, config=True)
326 326 readline_remove_delims = Unicode('-/~', config=True)
327 327 # don't use \M- bindings by default, because they
328 328 # conflict with 8-bit encodings. See gh-58,gh-88
329 329 readline_parse_and_bind = List([
330 330 'tab: complete',
331 331 '"\C-l": clear-screen',
332 332 'set show-all-if-ambiguous on',
333 333 '"\C-o": tab-insert',
334 334 '"\C-r": reverse-search-history',
335 335 '"\C-s": forward-search-history',
336 336 '"\C-p": history-search-backward',
337 337 '"\C-n": history-search-forward',
338 338 '"\e[A": history-search-backward',
339 339 '"\e[B": history-search-forward',
340 340 '"\C-k": kill-line',
341 341 '"\C-u": unix-line-discard',
342 342 ], allow_none=False, config=True)
343 343
344 344 # TODO: this part of prompt management should be moved to the frontends.
345 345 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
346 346 separate_in = SeparateUnicode('\n', config=True)
347 347 separate_out = SeparateUnicode('', config=True)
348 348 separate_out2 = SeparateUnicode('', config=True)
349 349 wildcards_case_sensitive = CBool(True, config=True)
350 350 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
351 351 default_value='Context', config=True)
352 352
353 353 # Subcomponents of InteractiveShell
354 354 alias_manager = Instance('IPython.core.alias.AliasManager')
355 355 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
356 356 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
357 357 display_trap = Instance('IPython.core.display_trap.DisplayTrap')
358 358 extension_manager = Instance('IPython.core.extensions.ExtensionManager')
359 359 plugin_manager = Instance('IPython.core.plugin.PluginManager')
360 360 payload_manager = Instance('IPython.core.payload.PayloadManager')
361 361 history_manager = Instance('IPython.core.history.HistoryManager')
362 362
363 363 profile_dir = Instance('IPython.core.application.ProfileDir')
364 364 @property
365 365 def profile(self):
366 366 if self.profile_dir is not None:
367 367 name = os.path.basename(self.profile_dir.location)
368 368 return name.replace('profile_','')
369 369
370 370
371 371 # Private interface
372 372 _post_execute = Instance(dict)
373 373
374 374 def __init__(self, config=None, ipython_dir=None, profile_dir=None,
375 375 user_module=None, user_ns=None,
376 376 custom_exceptions=((), None)):
377 377
378 378 # This is where traits with a config_key argument are updated
379 379 # from the values on config.
380 380 super(InteractiveShell, self).__init__(config=config)
381 381 self.configurables = [self]
382 382
383 383 # These are relatively independent and stateless
384 384 self.init_ipython_dir(ipython_dir)
385 385 self.init_profile_dir(profile_dir)
386 386 self.init_instance_attrs()
387 387 self.init_environment()
388 388
389 389 # Create namespaces (user_ns, user_global_ns, etc.)
390 390 self.init_create_namespaces(user_module, user_ns)
391 391 # This has to be done after init_create_namespaces because it uses
392 392 # something in self.user_ns, but before init_sys_modules, which
393 393 # is the first thing to modify sys.
394 394 # TODO: When we override sys.stdout and sys.stderr before this class
395 395 # is created, we are saving the overridden ones here. Not sure if this
396 396 # is what we want to do.
397 397 self.save_sys_module_state()
398 398 self.init_sys_modules()
399 399
400 400 # While we're trying to have each part of the code directly access what
401 401 # it needs without keeping redundant references to objects, we have too
402 402 # much legacy code that expects ip.db to exist.
403 403 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
404 404
405 405 self.init_history()
406 406 self.init_encoding()
407 407 self.init_prefilter()
408 408
409 409 Magic.__init__(self, self)
410 410
411 411 self.init_syntax_highlighting()
412 412 self.init_hooks()
413 413 self.init_pushd_popd_magic()
414 414 # self.init_traceback_handlers use to be here, but we moved it below
415 415 # because it and init_io have to come after init_readline.
416 416 self.init_user_ns()
417 417 self.init_logger()
418 418 self.init_alias()
419 419 self.init_builtins()
420 420
421 421 # pre_config_initialization
422 422
423 423 # The next section should contain everything that was in ipmaker.
424 424 self.init_logstart()
425 425
426 426 # The following was in post_config_initialization
427 427 self.init_inspector()
428 428 # init_readline() must come before init_io(), because init_io uses
429 429 # readline related things.
430 430 self.init_readline()
431 431 # We save this here in case user code replaces raw_input, but it needs
432 432 # to be after init_readline(), because PyPy's readline works by replacing
433 433 # raw_input.
434 434 if py3compat.PY3:
435 435 self.raw_input_original = input
436 436 else:
437 437 self.raw_input_original = raw_input
438 438 # init_completer must come after init_readline, because it needs to
439 439 # know whether readline is present or not system-wide to configure the
440 440 # completers, since the completion machinery can now operate
441 441 # independently of readline (e.g. over the network)
442 442 self.init_completer()
443 443 # TODO: init_io() needs to happen before init_traceback handlers
444 444 # because the traceback handlers hardcode the stdout/stderr streams.
445 445 # This logic in in debugger.Pdb and should eventually be changed.
446 446 self.init_io()
447 447 self.init_traceback_handlers(custom_exceptions)
448 448 self.init_prompts()
449 449 self.init_display_formatter()
450 450 self.init_display_pub()
451 451 self.init_displayhook()
452 452 self.init_reload_doctest()
453 453 self.init_magics()
454 454 self.init_pdb()
455 455 self.init_extension_manager()
456 456 self.init_plugin_manager()
457 457 self.init_payload()
458 458 self.hooks.late_startup_hook()
459 459 atexit.register(self.atexit_operations)
460 460
461 461 def get_ipython(self):
462 462 """Return the currently running IPython instance."""
463 463 return self
464 464
465 465 #-------------------------------------------------------------------------
466 466 # Trait changed handlers
467 467 #-------------------------------------------------------------------------
468 468
469 469 def _ipython_dir_changed(self, name, new):
470 470 if not os.path.isdir(new):
471 471 os.makedirs(new, mode = 0777)
472 472
473 473 def set_autoindent(self,value=None):
474 474 """Set the autoindent flag, checking for readline support.
475 475
476 476 If called with no arguments, it acts as a toggle."""
477 477
478 478 if value != 0 and not self.has_readline:
479 479 if os.name == 'posix':
480 480 warn("The auto-indent feature requires the readline library")
481 481 self.autoindent = 0
482 482 return
483 483 if value is None:
484 484 self.autoindent = not self.autoindent
485 485 else:
486 486 self.autoindent = value
487 487
488 488 #-------------------------------------------------------------------------
489 489 # init_* methods called by __init__
490 490 #-------------------------------------------------------------------------
491 491
492 492 def init_ipython_dir(self, ipython_dir):
493 493 if ipython_dir is not None:
494 494 self.ipython_dir = ipython_dir
495 495 return
496 496
497 497 self.ipython_dir = get_ipython_dir()
498 498
499 499 def init_profile_dir(self, profile_dir):
500 500 if profile_dir is not None:
501 501 self.profile_dir = profile_dir
502 502 return
503 503 self.profile_dir =\
504 504 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
505 505
506 506 def init_instance_attrs(self):
507 507 self.more = False
508 508
509 509 # command compiler
510 510 self.compile = CachingCompiler()
511 511
512 512 # Make an empty namespace, which extension writers can rely on both
513 513 # existing and NEVER being used by ipython itself. This gives them a
514 514 # convenient location for storing additional information and state
515 515 # their extensions may require, without fear of collisions with other
516 516 # ipython names that may develop later.
517 517 self.meta = Struct()
518 518
519 519 # Temporary files used for various purposes. Deleted at exit.
520 520 self.tempfiles = []
521 521
522 522 # Keep track of readline usage (later set by init_readline)
523 523 self.has_readline = False
524 524
525 525 # keep track of where we started running (mainly for crash post-mortem)
526 526 # This is not being used anywhere currently.
527 527 self.starting_dir = os.getcwdu()
528 528
529 529 # Indentation management
530 530 self.indent_current_nsp = 0
531 531
532 532 # Dict to track post-execution functions that have been registered
533 533 self._post_execute = {}
534 534
535 535 def init_environment(self):
536 536 """Any changes we need to make to the user's environment."""
537 537 pass
538 538
539 539 def init_encoding(self):
540 540 # Get system encoding at startup time. Certain terminals (like Emacs
541 541 # under Win32 have it set to None, and we need to have a known valid
542 542 # encoding to use in the raw_input() method
543 543 try:
544 544 self.stdin_encoding = sys.stdin.encoding or 'ascii'
545 545 except AttributeError:
546 546 self.stdin_encoding = 'ascii'
547 547
548 548 def init_syntax_highlighting(self):
549 549 # Python source parser/formatter for syntax highlighting
550 550 pyformat = PyColorize.Parser().format
551 551 self.pycolorize = lambda src: pyformat(src,'str',self.colors)
552 552
553 553 def init_pushd_popd_magic(self):
554 554 # for pushd/popd management
555 555 self.home_dir = get_home_dir()
556 556
557 557 self.dir_stack = []
558 558
559 559 def init_logger(self):
560 560 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
561 561 logmode='rotate')
562 562
563 563 def init_logstart(self):
564 564 """Initialize logging in case it was requested at the command line.
565 565 """
566 566 if self.logappend:
567 567 self.magic_logstart(self.logappend + ' append')
568 568 elif self.logfile:
569 569 self.magic_logstart(self.logfile)
570 570 elif self.logstart:
571 571 self.magic_logstart()
572 572
573 573 def init_builtins(self):
574 574 self.builtin_trap = BuiltinTrap(shell=self)
575 575
576 576 def init_inspector(self):
577 577 # Object inspector
578 578 self.inspector = oinspect.Inspector(oinspect.InspectColors,
579 579 PyColorize.ANSICodeColors,
580 580 'NoColor',
581 581 self.object_info_string_level)
582 582
583 583 def init_io(self):
584 584 # This will just use sys.stdout and sys.stderr. If you want to
585 585 # override sys.stdout and sys.stderr themselves, you need to do that
586 586 # *before* instantiating this class, because io holds onto
587 587 # references to the underlying streams.
588 588 if sys.platform == 'win32' and self.has_readline:
589 589 io.stdout = io.stderr = io.IOStream(self.readline._outputfile)
590 590 else:
591 591 io.stdout = io.IOStream(sys.stdout)
592 592 io.stderr = io.IOStream(sys.stderr)
593 593
594 594 def init_prompts(self):
595 595 # TODO: This is a pass for now because the prompts are managed inside
596 596 # the DisplayHook. Once there is a separate prompt manager, this
597 597 # will initialize that object and all prompt related information.
598 598 pass
599 599
600 600 def init_display_formatter(self):
601 601 self.display_formatter = DisplayFormatter(config=self.config)
602 602 self.configurables.append(self.display_formatter)
603 603
604 604 def init_display_pub(self):
605 605 self.display_pub = self.display_pub_class(config=self.config)
606 606 self.configurables.append(self.display_pub)
607 607
608 608 def init_displayhook(self):
609 609 # Initialize displayhook, set in/out prompts and printing system
610 610 self.displayhook = self.displayhook_class(
611 611 config=self.config,
612 612 shell=self,
613 613 cache_size=self.cache_size,
614 614 input_sep = self.separate_in,
615 615 output_sep = self.separate_out,
616 616 output_sep2 = self.separate_out2,
617 617 ps1 = self.prompt_in1,
618 618 ps2 = self.prompt_in2,
619 619 ps_out = self.prompt_out,
620 620 pad_left = self.prompts_pad_left
621 621 )
622 622 self.configurables.append(self.displayhook)
623 623 # This is a context manager that installs/revmoes the displayhook at
624 624 # the appropriate time.
625 625 self.display_trap = DisplayTrap(hook=self.displayhook)
626 626
627 627 def init_reload_doctest(self):
628 628 # Do a proper resetting of doctest, including the necessary displayhook
629 629 # monkeypatching
630 630 try:
631 631 doctest_reload()
632 632 except ImportError:
633 633 warn("doctest module does not exist.")
634 634
635 635 #-------------------------------------------------------------------------
636 636 # Things related to injections into the sys module
637 637 #-------------------------------------------------------------------------
638 638
639 639 def save_sys_module_state(self):
640 640 """Save the state of hooks in the sys module.
641 641
642 642 This has to be called after self.user_module is created.
643 643 """
644 644 self._orig_sys_module_state = {}
645 645 self._orig_sys_module_state['stdin'] = sys.stdin
646 646 self._orig_sys_module_state['stdout'] = sys.stdout
647 647 self._orig_sys_module_state['stderr'] = sys.stderr
648 648 self._orig_sys_module_state['excepthook'] = sys.excepthook
649 649 self._orig_sys_modules_main_name = self.user_module.__name__
650 650
651 651 def restore_sys_module_state(self):
652 652 """Restore the state of the sys module."""
653 653 try:
654 654 for k, v in self._orig_sys_module_state.iteritems():
655 655 setattr(sys, k, v)
656 656 except AttributeError:
657 657 pass
658 658 # Reset what what done in self.init_sys_modules
659 659 sys.modules[self.user_module.__name__] = self._orig_sys_modules_main_name
660 660
661 661 #-------------------------------------------------------------------------
662 662 # Things related to hooks
663 663 #-------------------------------------------------------------------------
664 664
665 665 def init_hooks(self):
666 666 # hooks holds pointers used for user-side customizations
667 667 self.hooks = Struct()
668 668
669 669 self.strdispatchers = {}
670 670
671 671 # Set all default hooks, defined in the IPython.hooks module.
672 672 hooks = IPython.core.hooks
673 673 for hook_name in hooks.__all__:
674 674 # default hooks have priority 100, i.e. low; user hooks should have
675 675 # 0-100 priority
676 676 self.set_hook(hook_name,getattr(hooks,hook_name), 100)
677 677
678 678 def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
679 679 """set_hook(name,hook) -> sets an internal IPython hook.
680 680
681 681 IPython exposes some of its internal API as user-modifiable hooks. By
682 682 adding your function to one of these hooks, you can modify IPython's
683 683 behavior to call at runtime your own routines."""
684 684
685 685 # At some point in the future, this should validate the hook before it
686 686 # accepts it. Probably at least check that the hook takes the number
687 687 # of args it's supposed to.
688 688
689 689 f = types.MethodType(hook,self)
690 690
691 691 # check if the hook is for strdispatcher first
692 692 if str_key is not None:
693 693 sdp = self.strdispatchers.get(name, StrDispatch())
694 694 sdp.add_s(str_key, f, priority )
695 695 self.strdispatchers[name] = sdp
696 696 return
697 697 if re_key is not None:
698 698 sdp = self.strdispatchers.get(name, StrDispatch())
699 699 sdp.add_re(re.compile(re_key), f, priority )
700 700 self.strdispatchers[name] = sdp
701 701 return
702 702
703 703 dp = getattr(self.hooks, name, None)
704 704 if name not in IPython.core.hooks.__all__:
705 705 print "Warning! Hook '%s' is not one of %s" % \
706 706 (name, IPython.core.hooks.__all__ )
707 707 if not dp:
708 708 dp = IPython.core.hooks.CommandChainDispatcher()
709 709
710 710 try:
711 711 dp.add(f,priority)
712 712 except AttributeError:
713 713 # it was not commandchain, plain old func - replace
714 714 dp = f
715 715
716 716 setattr(self.hooks,name, dp)
717 717
718 718 def register_post_execute(self, func):
719 719 """Register a function for calling after code execution.
720 720 """
721 721 if not callable(func):
722 722 raise ValueError('argument %s must be callable' % func)
723 723 self._post_execute[func] = True
724 724
725 725 #-------------------------------------------------------------------------
726 726 # Things related to the "main" module
727 727 #-------------------------------------------------------------------------
728 728
729 729 def new_main_mod(self,ns=None):
730 730 """Return a new 'main' module object for user code execution.
731 731 """
732 732 main_mod = self._user_main_module
733 733 init_fakemod_dict(main_mod,ns)
734 734 return main_mod
735 735
736 736 def cache_main_mod(self,ns,fname):
737 737 """Cache a main module's namespace.
738 738
739 739 When scripts are executed via %run, we must keep a reference to the
740 740 namespace of their __main__ module (a FakeModule instance) around so
741 741 that Python doesn't clear it, rendering objects defined therein
742 742 useless.
743 743
744 744 This method keeps said reference in a private dict, keyed by the
745 745 absolute path of the module object (which corresponds to the script
746 746 path). This way, for multiple executions of the same script we only
747 747 keep one copy of the namespace (the last one), thus preventing memory
748 748 leaks from old references while allowing the objects from the last
749 749 execution to be accessible.
750 750
751 751 Note: we can not allow the actual FakeModule instances to be deleted,
752 752 because of how Python tears down modules (it hard-sets all their
753 753 references to None without regard for reference counts). This method
754 754 must therefore make a *copy* of the given namespace, to allow the
755 755 original module's __dict__ to be cleared and reused.
756 756
757 757
758 758 Parameters
759 759 ----------
760 760 ns : a namespace (a dict, typically)
761 761
762 762 fname : str
763 763 Filename associated with the namespace.
764 764
765 765 Examples
766 766 --------
767 767
768 768 In [10]: import IPython
769 769
770 770 In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
771 771
772 772 In [12]: IPython.__file__ in _ip._main_ns_cache
773 773 Out[12]: True
774 774 """
775 775 self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
776 776
777 777 def clear_main_mod_cache(self):
778 778 """Clear the cache of main modules.
779 779
780 780 Mainly for use by utilities like %reset.
781 781
782 782 Examples
783 783 --------
784 784
785 785 In [15]: import IPython
786 786
787 787 In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
788 788
789 789 In [17]: len(_ip._main_ns_cache) > 0
790 790 Out[17]: True
791 791
792 792 In [18]: _ip.clear_main_mod_cache()
793 793
794 794 In [19]: len(_ip._main_ns_cache) == 0
795 795 Out[19]: True
796 796 """
797 797 self._main_ns_cache.clear()
798 798
799 799 #-------------------------------------------------------------------------
800 800 # Things related to debugging
801 801 #-------------------------------------------------------------------------
802 802
803 803 def init_pdb(self):
804 804 # Set calling of pdb on exceptions
805 805 # self.call_pdb is a property
806 806 self.call_pdb = self.pdb
807 807
808 808 def _get_call_pdb(self):
809 809 return self._call_pdb
810 810
811 811 def _set_call_pdb(self,val):
812 812
813 813 if val not in (0,1,False,True):
814 814 raise ValueError,'new call_pdb value must be boolean'
815 815
816 816 # store value in instance
817 817 self._call_pdb = val
818 818
819 819 # notify the actual exception handlers
820 820 self.InteractiveTB.call_pdb = val
821 821
822 822 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
823 823 'Control auto-activation of pdb at exceptions')
824 824
825 825 def debugger(self,force=False):
826 826 """Call the pydb/pdb debugger.
827 827
828 828 Keywords:
829 829
830 830 - force(False): by default, this routine checks the instance call_pdb
831 831 flag and does not actually invoke the debugger if the flag is false.
832 832 The 'force' option forces the debugger to activate even if the flag
833 833 is false.
834 834 """
835 835
836 836 if not (force or self.call_pdb):
837 837 return
838 838
839 839 if not hasattr(sys,'last_traceback'):
840 840 error('No traceback has been produced, nothing to debug.')
841 841 return
842 842
843 843 # use pydb if available
844 844 if debugger.has_pydb:
845 845 from pydb import pm
846 846 else:
847 847 # fallback to our internal debugger
848 848 pm = lambda : self.InteractiveTB.debugger(force=True)
849 849
850 850 with self.readline_no_record:
851 851 pm()
852 852
853 853 #-------------------------------------------------------------------------
854 854 # Things related to IPython's various namespaces
855 855 #-------------------------------------------------------------------------
856 856
857 857 def init_create_namespaces(self, user_module=None, user_ns=None):
858 858 # Create the namespace where the user will operate. user_ns is
859 859 # normally the only one used, and it is passed to the exec calls as
860 860 # the locals argument. But we do carry a user_global_ns namespace
861 861 # given as the exec 'globals' argument, This is useful in embedding
862 862 # situations where the ipython shell opens in a context where the
863 863 # distinction between locals and globals is meaningful. For
864 864 # non-embedded contexts, it is just the same object as the user_ns dict.
865 865
866 866 # FIXME. For some strange reason, __builtins__ is showing up at user
867 867 # level as a dict instead of a module. This is a manual fix, but I
868 868 # should really track down where the problem is coming from. Alex
869 869 # Schmolck reported this problem first.
870 870
871 871 # A useful post by Alex Martelli on this topic:
872 872 # Re: inconsistent value from __builtins__
873 873 # Von: Alex Martelli <aleaxit@yahoo.com>
874 874 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
875 875 # Gruppen: comp.lang.python
876 876
877 877 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
878 878 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
879 879 # > <type 'dict'>
880 880 # > >>> print type(__builtins__)
881 881 # > <type 'module'>
882 882 # > Is this difference in return value intentional?
883 883
884 884 # Well, it's documented that '__builtins__' can be either a dictionary
885 885 # or a module, and it's been that way for a long time. Whether it's
886 886 # intentional (or sensible), I don't know. In any case, the idea is
887 887 # that if you need to access the built-in namespace directly, you
888 888 # should start with "import __builtin__" (note, no 's') which will
889 889 # definitely give you a module. Yeah, it's somewhat confusing:-(.
890 890
891 # These routines return properly built dicts as needed by the rest of
892 # the code, and can also be used by extension writers to generate
893 # properly initialized namespaces.
894 self.user_module = self.prepare_user_module(user_module)
891 # These routines return a properly built module and dict as needed by
892 # the rest of the code, and can also be used by extension writers to
893 # generate properly initialized namespaces.
894 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
895 895
896 if user_ns is None:
897 user_ns = self.user_module.__dict__
898 self.user_ns = user_ns
899
900 # An auxiliary namespace that checks what parts of the user_ns were
901 # loaded at startup, so we can list later only variables defined in
902 # actual interactive use. Since it is always a subset of user_ns, it
903 # doesn't need to be separately tracked in the ns_table.
896 # A record of hidden variables we have added to the user namespace, so
897 # we can list later only variables defined in actual interactive use.
904 898 self.user_ns_hidden = set()
905 899
906 900 # Now that FakeModule produces a real module, we've run into a nasty
907 901 # problem: after script execution (via %run), the module where the user
908 902 # code ran is deleted. Now that this object is a true module (needed
909 903 # so docetst and other tools work correctly), the Python module
910 904 # teardown mechanism runs over it, and sets to None every variable
911 905 # present in that module. Top-level references to objects from the
912 906 # script survive, because the user_ns is updated with them. However,
913 907 # calling functions defined in the script that use other things from
914 908 # the script will fail, because the function's closure had references
915 909 # to the original objects, which are now all None. So we must protect
916 910 # these modules from deletion by keeping a cache.
917 911 #
918 912 # To avoid keeping stale modules around (we only need the one from the
919 913 # last run), we use a dict keyed with the full path to the script, so
920 914 # only the last version of the module is held in the cache. Note,
921 915 # however, that we must cache the module *namespace contents* (their
922 916 # __dict__). Because if we try to cache the actual modules, old ones
923 917 # (uncached) could be destroyed while still holding references (such as
924 918 # those held by GUI objects that tend to be long-lived)>
925 919 #
926 920 # The %reset command will flush this cache. See the cache_main_mod()
927 921 # and clear_main_mod_cache() methods for details on use.
928 922
929 923 # This is the cache used for 'main' namespaces
930 924 self._main_ns_cache = {}
931 925 # And this is the single instance of FakeModule whose __dict__ we keep
932 926 # copying and clearing for reuse on each %run
933 927 self._user_main_module = FakeModule()
934 928
935 929 # A table holding all the namespaces IPython deals with, so that
936 930 # introspection facilities can search easily.
937 931 self.ns_table = {'user_global':self.user_module.__dict__,
938 932 'user_local':user_ns,
939 933 'builtin':builtin_mod.__dict__
940 934 }
941 935
942 936 @property
943 937 def user_global_ns(self):
944 938 return self.user_module.__dict__
945 939
946 def prepare_user_module(self, user_module=None):
947 """Prepares a module for use as the interactive __main__ module in
948 which user code is run.
940 def prepare_user_module(self, user_module=None, user_ns=None):
941 """Prepare the module and namespace in which user code will be run.
942
943 When IPython is started normally, both parameters are None: a new module
944 is created automatically, and its __dict__ used as the namespace.
945
946 If only user_module is provided, its __dict__ is used as the namespace.
947 If only user_ns is provided, a dummy module is created, and user_ns
948 becomes the global namespace. If both are provided (as they may be
949 when embedding), user_ns is the local namespace, and user_module
950 provides the global namespace.
949 951
950 952 Parameters
951 953 ----------
952 954 user_module : module, optional
953 955 The current user module in which IPython is being run. If None,
954 956 a clean module will be created.
957 user_ns : dict, optional
958 A namespace in which to run interactive commands.
955 959
956 960 Returns
957 961 -------
958 A module object.
962 A tuple of user_module and user_ns, each properly initialised.
959 963 """
964 if user_module is None and user_ns is not None:
965 user_ns.setdefault("__name__", "__main__")
966 class DummyMod(object):
967 "A dummy module used for IPython's interactive namespace."
968 pass
969 user_module = DummyMod()
970 user_module.__dict__ = user_ns
971
960 972 if user_module is None:
961 973 user_module = types.ModuleType("__main__",
962 974 doc="Automatically created module for IPython interactive environment")
963 975
964 976 # We must ensure that __builtin__ (without the final 's') is always
965 977 # available and pointing to the __builtin__ *module*. For more details:
966 978 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
967 979 user_module.__dict__.setdefault('__builtin__',__builtin__)
968 980 user_module.__dict__.setdefault('__builtins__',__builtin__)
981
982 if user_ns is None:
983 user_ns = user_module.__dict__
969 984
970 return user_module
985 return user_module, user_ns
971 986
972 987 def init_sys_modules(self):
973 988 # We need to insert into sys.modules something that looks like a
974 989 # module but which accesses the IPython namespace, for shelve and
975 990 # pickle to work interactively. Normally they rely on getting
976 991 # everything out of __main__, but for embedding purposes each IPython
977 992 # instance has its own private namespace, so we can't go shoving
978 993 # everything into __main__.
979 994
980 995 # note, however, that we should only do this for non-embedded
981 996 # ipythons, which really mimic the __main__.__dict__ with their own
982 997 # namespace. Embedded instances, on the other hand, should not do
983 998 # this because they need to manage the user local/global namespaces
984 999 # only, but they live within a 'normal' __main__ (meaning, they
985 1000 # shouldn't overtake the execution environment of the script they're
986 1001 # embedded in).
987 1002
988 1003 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
989 1004 main_name = self.user_module.__name__
990 1005 sys.modules[main_name] = self.user_module
991 1006
992 1007 def init_user_ns(self):
993 1008 """Initialize all user-visible namespaces to their minimum defaults.
994 1009
995 1010 Certain history lists are also initialized here, as they effectively
996 1011 act as user namespaces.
997 1012
998 1013 Notes
999 1014 -----
1000 1015 All data structures here are only filled in, they are NOT reset by this
1001 1016 method. If they were not empty before, data will simply be added to
1002 1017 therm.
1003 1018 """
1004 1019 # This function works in two parts: first we put a few things in
1005 1020 # user_ns, and we sync that contents into user_ns_hidden so that these
1006 1021 # initial variables aren't shown by %who. After the sync, we add the
1007 1022 # rest of what we *do* want the user to see with %who even on a new
1008 1023 # session (probably nothing, so theye really only see their own stuff)
1009 1024
1010 1025 # The user dict must *always* have a __builtin__ reference to the
1011 1026 # Python standard __builtin__ namespace, which must be imported.
1012 1027 # This is so that certain operations in prompt evaluation can be
1013 1028 # reliably executed with builtins. Note that we can NOT use
1014 1029 # __builtins__ (note the 's'), because that can either be a dict or a
1015 1030 # module, and can even mutate at runtime, depending on the context
1016 1031 # (Python makes no guarantees on it). In contrast, __builtin__ is
1017 1032 # always a module object, though it must be explicitly imported.
1018 1033
1019 1034 # For more details:
1020 1035 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1021 1036 ns = dict()
1022 1037
1023 1038 # Put 'help' in the user namespace
1024 1039 try:
1025 1040 from site import _Helper
1026 1041 ns['help'] = _Helper()
1027 1042 except ImportError:
1028 1043 warn('help() not available - check site.py')
1029 1044
1030 1045 # make global variables for user access to the histories
1031 1046 ns['_ih'] = self.history_manager.input_hist_parsed
1032 1047 ns['_oh'] = self.history_manager.output_hist
1033 1048 ns['_dh'] = self.history_manager.dir_hist
1034 1049
1035 1050 ns['_sh'] = shadowns
1036 1051
1037 1052 # user aliases to input and output histories. These shouldn't show up
1038 1053 # in %who, as they can have very large reprs.
1039 1054 ns['In'] = self.history_manager.input_hist_parsed
1040 1055 ns['Out'] = self.history_manager.output_hist
1041 1056
1042 1057 # Store myself as the public api!!!
1043 1058 ns['get_ipython'] = self.get_ipython
1044 1059
1045 1060 ns['exit'] = self.exiter
1046 1061 ns['quit'] = self.exiter
1047 1062
1048 1063 # Sync what we've added so far to user_ns_hidden so these aren't seen
1049 1064 # by %who
1050 1065 self.user_ns_hidden.update(ns)
1051 1066
1052 1067 # Anything put into ns now would show up in %who. Think twice before
1053 1068 # putting anything here, as we really want %who to show the user their
1054 1069 # stuff, not our variables.
1055 1070
1056 1071 # Finally, update the real user's namespace
1057 1072 self.user_ns.update(ns)
1058 1073
1059 1074 def reset(self, new_session=True):
1060 1075 """Clear all internal namespaces, and attempt to release references to
1061 1076 user objects.
1062 1077
1063 1078 If new_session is True, a new history session will be opened.
1064 1079 """
1065 1080 # Clear histories
1066 1081 self.history_manager.reset(new_session)
1067 1082 # Reset counter used to index all histories
1068 1083 if new_session:
1069 1084 self.execution_count = 1
1070 1085
1071 1086 # Flush cached output items
1072 1087 if self.displayhook.do_full_cache:
1073 1088 self.displayhook.flush()
1074 1089
1075 1090 # The main execution namespaces must be cleared very carefully,
1076 1091 # skipping the deletion of the builtin-related keys, because doing so
1077 1092 # would cause errors in many object's __del__ methods.
1078 for ns in [self.user_ns, self.user_global_ns]:
1079 drop_keys = set(ns.keys())
1080 drop_keys.discard('__builtin__')
1081 drop_keys.discard('__builtins__')
1082 for k in drop_keys:
1083 del ns[k]
1084
1093 if self.user_ns is not self.user_global_ns:
1094 self.user_ns.clear()
1095 ns = self.user_global_ns
1096 drop_keys = set(ns.keys())
1097 drop_keys.discard('__builtin__')
1098 drop_keys.discard('__builtins__')
1099 drop_keys.discard('__name__')
1100 for k in drop_keys:
1101 del ns[k]
1102
1085 1103 # Restore the user namespaces to minimal usability
1086 1104 self.init_user_ns()
1087 1105
1088 1106 # Restore the default and user aliases
1089 1107 self.alias_manager.clear_aliases()
1090 1108 self.alias_manager.init_aliases()
1091 1109
1092 1110 # Flush the private list of module references kept for script
1093 1111 # execution protection
1094 1112 self.clear_main_mod_cache()
1095 1113
1096 1114 # Clear out the namespace from the last %run
1097 1115 self.new_main_mod()
1098 1116
1099 1117 def del_var(self, varname, by_name=False):
1100 1118 """Delete a variable from the various namespaces, so that, as
1101 1119 far as possible, we're not keeping any hidden references to it.
1102 1120
1103 1121 Parameters
1104 1122 ----------
1105 1123 varname : str
1106 1124 The name of the variable to delete.
1107 1125 by_name : bool
1108 1126 If True, delete variables with the given name in each
1109 1127 namespace. If False (default), find the variable in the user
1110 1128 namespace, and delete references to it.
1111 1129 """
1112 1130 if varname in ('__builtin__', '__builtins__'):
1113 1131 raise ValueError("Refusing to delete %s" % varname)
1114 1132
1115 1133 ns_refs = [self.user_ns, self.user_global_ns,
1116 1134 self._user_main_module.__dict__] + self._main_ns_cache.values()
1117 1135
1118 1136 if by_name: # Delete by name
1119 1137 for ns in ns_refs:
1120 1138 try:
1121 1139 del ns[varname]
1122 1140 except KeyError:
1123 1141 pass
1124 1142 else: # Delete by object
1125 1143 try:
1126 1144 obj = self.user_ns[varname]
1127 1145 except KeyError:
1128 1146 raise NameError("name '%s' is not defined" % varname)
1129 1147 # Also check in output history
1130 1148 ns_refs.append(self.history_manager.output_hist)
1131 1149 for ns in ns_refs:
1132 1150 to_delete = [n for n, o in ns.iteritems() if o is obj]
1133 1151 for name in to_delete:
1134 1152 del ns[name]
1135 1153
1136 1154 # displayhook keeps extra references, but not in a dictionary
1137 1155 for name in ('_', '__', '___'):
1138 1156 if getattr(self.displayhook, name) is obj:
1139 1157 setattr(self.displayhook, name, None)
1140 1158
1141 1159 def reset_selective(self, regex=None):
1142 1160 """Clear selective variables from internal namespaces based on a
1143 1161 specified regular expression.
1144 1162
1145 1163 Parameters
1146 1164 ----------
1147 1165 regex : string or compiled pattern, optional
1148 1166 A regular expression pattern that will be used in searching
1149 1167 variable names in the users namespaces.
1150 1168 """
1151 1169 if regex is not None:
1152 1170 try:
1153 1171 m = re.compile(regex)
1154 1172 except TypeError:
1155 1173 raise TypeError('regex must be a string or compiled pattern')
1156 1174 # Search for keys in each namespace that match the given regex
1157 1175 # If a match is found, delete the key/value pair.
1158 1176 for ns in [self.user_ns, self.user_global_ns]:
1159 1177 for var in ns:
1160 1178 if m.search(var):
1161 1179 del ns[var]
1162 1180
1163 1181 def push(self, variables, interactive=True):
1164 1182 """Inject a group of variables into the IPython user namespace.
1165 1183
1166 1184 Parameters
1167 1185 ----------
1168 1186 variables : dict, str or list/tuple of str
1169 1187 The variables to inject into the user's namespace. If a dict, a
1170 1188 simple update is done. If a str, the string is assumed to have
1171 1189 variable names separated by spaces. A list/tuple of str can also
1172 1190 be used to give the variable names. If just the variable names are
1173 1191 give (list/tuple/str) then the variable values looked up in the
1174 1192 callers frame.
1175 1193 interactive : bool
1176 1194 If True (default), the variables will be listed with the ``who``
1177 1195 magic.
1178 1196 """
1179 1197 vdict = None
1180 1198
1181 1199 # We need a dict of name/value pairs to do namespace updates.
1182 1200 if isinstance(variables, dict):
1183 1201 vdict = variables
1184 1202 elif isinstance(variables, (basestring, list, tuple)):
1185 1203 if isinstance(variables, basestring):
1186 1204 vlist = variables.split()
1187 1205 else:
1188 1206 vlist = variables
1189 1207 vdict = {}
1190 1208 cf = sys._getframe(1)
1191 1209 for name in vlist:
1192 1210 try:
1193 1211 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1194 1212 except:
1195 1213 print ('Could not get variable %s from %s' %
1196 1214 (name,cf.f_code.co_name))
1197 1215 else:
1198 1216 raise ValueError('variables must be a dict/str/list/tuple')
1199 1217
1200 1218 # Propagate variables to user namespace
1201 1219 self.user_ns.update(vdict)
1202 1220
1203 1221 # And configure interactive visibility
1204 1222 user_ns_hidden = self.user_ns_hidden
1205 1223 if interactive:
1206 1224 for name in vdict:
1207 1225 user_ns_hidden.discard(name)
1208 1226 else:
1209 1227 for name in vdict:
1210 1228 user_ns_hidden.add(name)
1211 1229
1212 1230 def drop_by_id(self, variables):
1213 1231 """Remove a dict of variables from the user namespace, if they are the
1214 1232 same as the values in the dictionary.
1215 1233
1216 1234 This is intended for use by extensions: variables that they've added can
1217 1235 be taken back out if they are unloaded, without removing any that the
1218 1236 user has overwritten.
1219 1237
1220 1238 Parameters
1221 1239 ----------
1222 1240 variables : dict
1223 1241 A dictionary mapping object names (as strings) to the objects.
1224 1242 """
1225 1243 for name, obj in variables.iteritems():
1226 1244 if name in self.user_ns and self.user_ns[name] is obj:
1227 1245 del self.user_ns[name]
1228 1246 self.user_ns_hidden.discard(name)
1229 1247
1230 1248 #-------------------------------------------------------------------------
1231 1249 # Things related to object introspection
1232 1250 #-------------------------------------------------------------------------
1233 1251
1234 1252 def _ofind(self, oname, namespaces=None):
1235 1253 """Find an object in the available namespaces.
1236 1254
1237 1255 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1238 1256
1239 1257 Has special code to detect magic functions.
1240 1258 """
1241 1259 oname = oname.strip()
1242 1260 #print '1- oname: <%r>' % oname # dbg
1243 1261 if not py3compat.isidentifier(oname.lstrip(ESC_MAGIC), dotted=True):
1244 1262 return dict(found=False)
1245 1263
1246 1264 alias_ns = None
1247 1265 if namespaces is None:
1248 1266 # Namespaces to search in:
1249 1267 # Put them in a list. The order is important so that we
1250 1268 # find things in the same order that Python finds them.
1251 1269 namespaces = [ ('Interactive', self.user_ns),
1252 1270 ('Interactive (global)', self.user_global_ns),
1253 1271 ('Python builtin', builtin_mod.__dict__),
1254 1272 ('Alias', self.alias_manager.alias_table),
1255 1273 ]
1256 1274 alias_ns = self.alias_manager.alias_table
1257 1275
1258 1276 # initialize results to 'null'
1259 1277 found = False; obj = None; ospace = None; ds = None;
1260 1278 ismagic = False; isalias = False; parent = None
1261 1279
1262 1280 # We need to special-case 'print', which as of python2.6 registers as a
1263 1281 # function but should only be treated as one if print_function was
1264 1282 # loaded with a future import. In this case, just bail.
1265 1283 if (oname == 'print' and not py3compat.PY3 and not \
1266 1284 (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
1267 1285 return {'found':found, 'obj':obj, 'namespace':ospace,
1268 1286 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1269 1287
1270 1288 # Look for the given name by splitting it in parts. If the head is
1271 1289 # found, then we look for all the remaining parts as members, and only
1272 1290 # declare success if we can find them all.
1273 1291 oname_parts = oname.split('.')
1274 1292 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1275 1293 for nsname,ns in namespaces:
1276 1294 try:
1277 1295 obj = ns[oname_head]
1278 1296 except KeyError:
1279 1297 continue
1280 1298 else:
1281 1299 #print 'oname_rest:', oname_rest # dbg
1282 1300 for part in oname_rest:
1283 1301 try:
1284 1302 parent = obj
1285 1303 obj = getattr(obj,part)
1286 1304 except:
1287 1305 # Blanket except b/c some badly implemented objects
1288 1306 # allow __getattr__ to raise exceptions other than
1289 1307 # AttributeError, which then crashes IPython.
1290 1308 break
1291 1309 else:
1292 1310 # If we finish the for loop (no break), we got all members
1293 1311 found = True
1294 1312 ospace = nsname
1295 1313 if ns == alias_ns:
1296 1314 isalias = True
1297 1315 break # namespace loop
1298 1316
1299 1317 # Try to see if it's magic
1300 1318 if not found:
1301 1319 if oname.startswith(ESC_MAGIC):
1302 1320 oname = oname[1:]
1303 1321 obj = getattr(self,'magic_'+oname,None)
1304 1322 if obj is not None:
1305 1323 found = True
1306 1324 ospace = 'IPython internal'
1307 1325 ismagic = True
1308 1326
1309 1327 # Last try: special-case some literals like '', [], {}, etc:
1310 1328 if not found and oname_head in ["''",'""','[]','{}','()']:
1311 1329 obj = eval(oname_head)
1312 1330 found = True
1313 1331 ospace = 'Interactive'
1314 1332
1315 1333 return {'found':found, 'obj':obj, 'namespace':ospace,
1316 1334 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1317 1335
1318 1336 def _ofind_property(self, oname, info):
1319 1337 """Second part of object finding, to look for property details."""
1320 1338 if info.found:
1321 1339 # Get the docstring of the class property if it exists.
1322 1340 path = oname.split('.')
1323 1341 root = '.'.join(path[:-1])
1324 1342 if info.parent is not None:
1325 1343 try:
1326 1344 target = getattr(info.parent, '__class__')
1327 1345 # The object belongs to a class instance.
1328 1346 try:
1329 1347 target = getattr(target, path[-1])
1330 1348 # The class defines the object.
1331 1349 if isinstance(target, property):
1332 1350 oname = root + '.__class__.' + path[-1]
1333 1351 info = Struct(self._ofind(oname))
1334 1352 except AttributeError: pass
1335 1353 except AttributeError: pass
1336 1354
1337 1355 # We return either the new info or the unmodified input if the object
1338 1356 # hadn't been found
1339 1357 return info
1340 1358
1341 1359 def _object_find(self, oname, namespaces=None):
1342 1360 """Find an object and return a struct with info about it."""
1343 1361 inf = Struct(self._ofind(oname, namespaces))
1344 1362 return Struct(self._ofind_property(oname, inf))
1345 1363
1346 1364 def _inspect(self, meth, oname, namespaces=None, **kw):
1347 1365 """Generic interface to the inspector system.
1348 1366
1349 1367 This function is meant to be called by pdef, pdoc & friends."""
1350 1368 info = self._object_find(oname)
1351 1369 if info.found:
1352 1370 pmethod = getattr(self.inspector, meth)
1353 1371 formatter = format_screen if info.ismagic else None
1354 1372 if meth == 'pdoc':
1355 1373 pmethod(info.obj, oname, formatter)
1356 1374 elif meth == 'pinfo':
1357 1375 pmethod(info.obj, oname, formatter, info, **kw)
1358 1376 else:
1359 1377 pmethod(info.obj, oname)
1360 1378 else:
1361 1379 print 'Object `%s` not found.' % oname
1362 1380 return 'not found' # so callers can take other action
1363 1381
1364 1382 def object_inspect(self, oname):
1365 1383 with self.builtin_trap:
1366 1384 info = self._object_find(oname)
1367 1385 if info.found:
1368 1386 return self.inspector.info(info.obj, oname, info=info)
1369 1387 else:
1370 1388 return oinspect.object_info(name=oname, found=False)
1371 1389
1372 1390 #-------------------------------------------------------------------------
1373 1391 # Things related to history management
1374 1392 #-------------------------------------------------------------------------
1375 1393
1376 1394 def init_history(self):
1377 1395 """Sets up the command history, and starts regular autosaves."""
1378 1396 self.history_manager = HistoryManager(shell=self, config=self.config)
1379 1397 self.configurables.append(self.history_manager)
1380 1398
1381 1399 #-------------------------------------------------------------------------
1382 1400 # Things related to exception handling and tracebacks (not debugging)
1383 1401 #-------------------------------------------------------------------------
1384 1402
1385 1403 def init_traceback_handlers(self, custom_exceptions):
1386 1404 # Syntax error handler.
1387 1405 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
1388 1406
1389 1407 # The interactive one is initialized with an offset, meaning we always
1390 1408 # want to remove the topmost item in the traceback, which is our own
1391 1409 # internal code. Valid modes: ['Plain','Context','Verbose']
1392 1410 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1393 1411 color_scheme='NoColor',
1394 1412 tb_offset = 1,
1395 1413 check_cache=self.compile.check_cache)
1396 1414
1397 1415 # The instance will store a pointer to the system-wide exception hook,
1398 1416 # so that runtime code (such as magics) can access it. This is because
1399 1417 # during the read-eval loop, it may get temporarily overwritten.
1400 1418 self.sys_excepthook = sys.excepthook
1401 1419
1402 1420 # and add any custom exception handlers the user may have specified
1403 1421 self.set_custom_exc(*custom_exceptions)
1404 1422
1405 1423 # Set the exception mode
1406 1424 self.InteractiveTB.set_mode(mode=self.xmode)
1407 1425
1408 1426 def set_custom_exc(self, exc_tuple, handler):
1409 1427 """set_custom_exc(exc_tuple,handler)
1410 1428
1411 1429 Set a custom exception handler, which will be called if any of the
1412 1430 exceptions in exc_tuple occur in the mainloop (specifically, in the
1413 1431 run_code() method).
1414 1432
1415 1433 Parameters
1416 1434 ----------
1417 1435
1418 1436 exc_tuple : tuple of exception classes
1419 1437 A *tuple* of exception classes, for which to call the defined
1420 1438 handler. It is very important that you use a tuple, and NOT A
1421 1439 LIST here, because of the way Python's except statement works. If
1422 1440 you only want to trap a single exception, use a singleton tuple::
1423 1441
1424 1442 exc_tuple == (MyCustomException,)
1425 1443
1426 1444 handler : callable
1427 1445 handler must have the following signature::
1428 1446
1429 1447 def my_handler(self, etype, value, tb, tb_offset=None):
1430 1448 ...
1431 1449 return structured_traceback
1432 1450
1433 1451 Your handler must return a structured traceback (a list of strings),
1434 1452 or None.
1435 1453
1436 1454 This will be made into an instance method (via types.MethodType)
1437 1455 of IPython itself, and it will be called if any of the exceptions
1438 1456 listed in the exc_tuple are caught. If the handler is None, an
1439 1457 internal basic one is used, which just prints basic info.
1440 1458
1441 1459 To protect IPython from crashes, if your handler ever raises an
1442 1460 exception or returns an invalid result, it will be immediately
1443 1461 disabled.
1444 1462
1445 1463 WARNING: by putting in your own exception handler into IPython's main
1446 1464 execution loop, you run a very good chance of nasty crashes. This
1447 1465 facility should only be used if you really know what you are doing."""
1448 1466
1449 1467 assert type(exc_tuple)==type(()) , \
1450 1468 "The custom exceptions must be given AS A TUPLE."
1451 1469
1452 1470 def dummy_handler(self,etype,value,tb,tb_offset=None):
1453 1471 print '*** Simple custom exception handler ***'
1454 1472 print 'Exception type :',etype
1455 1473 print 'Exception value:',value
1456 1474 print 'Traceback :',tb
1457 1475 #print 'Source code :','\n'.join(self.buffer)
1458 1476
1459 1477 def validate_stb(stb):
1460 1478 """validate structured traceback return type
1461 1479
1462 1480 return type of CustomTB *should* be a list of strings, but allow
1463 1481 single strings or None, which are harmless.
1464 1482
1465 1483 This function will *always* return a list of strings,
1466 1484 and will raise a TypeError if stb is inappropriate.
1467 1485 """
1468 1486 msg = "CustomTB must return list of strings, not %r" % stb
1469 1487 if stb is None:
1470 1488 return []
1471 1489 elif isinstance(stb, basestring):
1472 1490 return [stb]
1473 1491 elif not isinstance(stb, list):
1474 1492 raise TypeError(msg)
1475 1493 # it's a list
1476 1494 for line in stb:
1477 1495 # check every element
1478 1496 if not isinstance(line, basestring):
1479 1497 raise TypeError(msg)
1480 1498 return stb
1481 1499
1482 1500 if handler is None:
1483 1501 wrapped = dummy_handler
1484 1502 else:
1485 1503 def wrapped(self,etype,value,tb,tb_offset=None):
1486 1504 """wrap CustomTB handler, to protect IPython from user code
1487 1505
1488 1506 This makes it harder (but not impossible) for custom exception
1489 1507 handlers to crash IPython.
1490 1508 """
1491 1509 try:
1492 1510 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1493 1511 return validate_stb(stb)
1494 1512 except:
1495 1513 # clear custom handler immediately
1496 1514 self.set_custom_exc((), None)
1497 1515 print >> io.stderr, "Custom TB Handler failed, unregistering"
1498 1516 # show the exception in handler first
1499 1517 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1500 1518 print >> io.stdout, self.InteractiveTB.stb2text(stb)
1501 1519 print >> io.stdout, "The original exception:"
1502 1520 stb = self.InteractiveTB.structured_traceback(
1503 1521 (etype,value,tb), tb_offset=tb_offset
1504 1522 )
1505 1523 return stb
1506 1524
1507 1525 self.CustomTB = types.MethodType(wrapped,self)
1508 1526 self.custom_exceptions = exc_tuple
1509 1527
1510 1528 def excepthook(self, etype, value, tb):
1511 1529 """One more defense for GUI apps that call sys.excepthook.
1512 1530
1513 1531 GUI frameworks like wxPython trap exceptions and call
1514 1532 sys.excepthook themselves. I guess this is a feature that
1515 1533 enables them to keep running after exceptions that would
1516 1534 otherwise kill their mainloop. This is a bother for IPython
1517 1535 which excepts to catch all of the program exceptions with a try:
1518 1536 except: statement.
1519 1537
1520 1538 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1521 1539 any app directly invokes sys.excepthook, it will look to the user like
1522 1540 IPython crashed. In order to work around this, we can disable the
1523 1541 CrashHandler and replace it with this excepthook instead, which prints a
1524 1542 regular traceback using our InteractiveTB. In this fashion, apps which
1525 1543 call sys.excepthook will generate a regular-looking exception from
1526 1544 IPython, and the CrashHandler will only be triggered by real IPython
1527 1545 crashes.
1528 1546
1529 1547 This hook should be used sparingly, only in places which are not likely
1530 1548 to be true IPython errors.
1531 1549 """
1532 1550 self.showtraceback((etype,value,tb),tb_offset=0)
1533 1551
1534 1552 def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
1535 1553 exception_only=False):
1536 1554 """Display the exception that just occurred.
1537 1555
1538 1556 If nothing is known about the exception, this is the method which
1539 1557 should be used throughout the code for presenting user tracebacks,
1540 1558 rather than directly invoking the InteractiveTB object.
1541 1559
1542 1560 A specific showsyntaxerror() also exists, but this method can take
1543 1561 care of calling it if needed, so unless you are explicitly catching a
1544 1562 SyntaxError exception, don't try to analyze the stack manually and
1545 1563 simply call this method."""
1546 1564
1547 1565 try:
1548 1566 if exc_tuple is None:
1549 1567 etype, value, tb = sys.exc_info()
1550 1568 else:
1551 1569 etype, value, tb = exc_tuple
1552 1570
1553 1571 if etype is None:
1554 1572 if hasattr(sys, 'last_type'):
1555 1573 etype, value, tb = sys.last_type, sys.last_value, \
1556 1574 sys.last_traceback
1557 1575 else:
1558 1576 self.write_err('No traceback available to show.\n')
1559 1577 return
1560 1578
1561 1579 if etype is SyntaxError:
1562 1580 # Though this won't be called by syntax errors in the input
1563 1581 # line, there may be SyntaxError cases with imported code.
1564 1582 self.showsyntaxerror(filename)
1565 1583 elif etype is UsageError:
1566 1584 self.write_err("UsageError: %s" % value)
1567 1585 else:
1568 1586 # WARNING: these variables are somewhat deprecated and not
1569 1587 # necessarily safe to use in a threaded environment, but tools
1570 1588 # like pdb depend on their existence, so let's set them. If we
1571 1589 # find problems in the field, we'll need to revisit their use.
1572 1590 sys.last_type = etype
1573 1591 sys.last_value = value
1574 1592 sys.last_traceback = tb
1575 1593 if etype in self.custom_exceptions:
1576 1594 stb = self.CustomTB(etype, value, tb, tb_offset)
1577 1595 else:
1578 1596 if exception_only:
1579 1597 stb = ['An exception has occurred, use %tb to see '
1580 1598 'the full traceback.\n']
1581 1599 stb.extend(self.InteractiveTB.get_exception_only(etype,
1582 1600 value))
1583 1601 else:
1584 1602 stb = self.InteractiveTB.structured_traceback(etype,
1585 1603 value, tb, tb_offset=tb_offset)
1586 1604
1587 1605 self._showtraceback(etype, value, stb)
1588 1606 if self.call_pdb:
1589 1607 # drop into debugger
1590 1608 self.debugger(force=True)
1591 1609 return
1592 1610
1593 1611 # Actually show the traceback
1594 1612 self._showtraceback(etype, value, stb)
1595 1613
1596 1614 except KeyboardInterrupt:
1597 1615 self.write_err("\nKeyboardInterrupt\n")
1598 1616
1599 1617 def _showtraceback(self, etype, evalue, stb):
1600 1618 """Actually show a traceback.
1601 1619
1602 1620 Subclasses may override this method to put the traceback on a different
1603 1621 place, like a side channel.
1604 1622 """
1605 1623 print >> io.stdout, self.InteractiveTB.stb2text(stb)
1606 1624
1607 1625 def showsyntaxerror(self, filename=None):
1608 1626 """Display the syntax error that just occurred.
1609 1627
1610 1628 This doesn't display a stack trace because there isn't one.
1611 1629
1612 1630 If a filename is given, it is stuffed in the exception instead
1613 1631 of what was there before (because Python's parser always uses
1614 1632 "<string>" when reading from a string).
1615 1633 """
1616 1634 etype, value, last_traceback = sys.exc_info()
1617 1635
1618 1636 # See note about these variables in showtraceback() above
1619 1637 sys.last_type = etype
1620 1638 sys.last_value = value
1621 1639 sys.last_traceback = last_traceback
1622 1640
1623 1641 if filename and etype is SyntaxError:
1624 1642 # Work hard to stuff the correct filename in the exception
1625 1643 try:
1626 1644 msg, (dummy_filename, lineno, offset, line) = value
1627 1645 except:
1628 1646 # Not the format we expect; leave it alone
1629 1647 pass
1630 1648 else:
1631 1649 # Stuff in the right filename
1632 1650 try:
1633 1651 # Assume SyntaxError is a class exception
1634 1652 value = SyntaxError(msg, (filename, lineno, offset, line))
1635 1653 except:
1636 1654 # If that failed, assume SyntaxError is a string
1637 1655 value = msg, (filename, lineno, offset, line)
1638 1656 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1639 1657 self._showtraceback(etype, value, stb)
1640 1658
1641 1659 # This is overridden in TerminalInteractiveShell to show a message about
1642 1660 # the %paste magic.
1643 1661 def showindentationerror(self):
1644 1662 """Called by run_cell when there's an IndentationError in code entered
1645 1663 at the prompt.
1646 1664
1647 1665 This is overridden in TerminalInteractiveShell to show a message about
1648 1666 the %paste magic."""
1649 1667 self.showsyntaxerror()
1650 1668
1651 1669 #-------------------------------------------------------------------------
1652 1670 # Things related to readline
1653 1671 #-------------------------------------------------------------------------
1654 1672
1655 1673 def init_readline(self):
1656 1674 """Command history completion/saving/reloading."""
1657 1675
1658 1676 if self.readline_use:
1659 1677 import IPython.utils.rlineimpl as readline
1660 1678
1661 1679 self.rl_next_input = None
1662 1680 self.rl_do_indent = False
1663 1681
1664 1682 if not self.readline_use or not readline.have_readline:
1665 1683 self.has_readline = False
1666 1684 self.readline = None
1667 1685 # Set a number of methods that depend on readline to be no-op
1668 1686 self.readline_no_record = no_op_context
1669 1687 self.set_readline_completer = no_op
1670 1688 self.set_custom_completer = no_op
1671 1689 self.set_completer_frame = no_op
1672 1690 if self.readline_use:
1673 1691 warn('Readline services not available or not loaded.')
1674 1692 else:
1675 1693 self.has_readline = True
1676 1694 self.readline = readline
1677 1695 sys.modules['readline'] = readline
1678 1696
1679 1697 # Platform-specific configuration
1680 1698 if os.name == 'nt':
1681 1699 # FIXME - check with Frederick to see if we can harmonize
1682 1700 # naming conventions with pyreadline to avoid this
1683 1701 # platform-dependent check
1684 1702 self.readline_startup_hook = readline.set_pre_input_hook
1685 1703 else:
1686 1704 self.readline_startup_hook = readline.set_startup_hook
1687 1705
1688 1706 # Load user's initrc file (readline config)
1689 1707 # Or if libedit is used, load editrc.
1690 1708 inputrc_name = os.environ.get('INPUTRC')
1691 1709 if inputrc_name is None:
1692 1710 inputrc_name = '.inputrc'
1693 1711 if readline.uses_libedit:
1694 1712 inputrc_name = '.editrc'
1695 1713 inputrc_name = os.path.join(self.home_dir, inputrc_name)
1696 1714 if os.path.isfile(inputrc_name):
1697 1715 try:
1698 1716 readline.read_init_file(inputrc_name)
1699 1717 except:
1700 1718 warn('Problems reading readline initialization file <%s>'
1701 1719 % inputrc_name)
1702 1720
1703 1721 # Configure readline according to user's prefs
1704 1722 # This is only done if GNU readline is being used. If libedit
1705 1723 # is being used (as on Leopard) the readline config is
1706 1724 # not run as the syntax for libedit is different.
1707 1725 if not readline.uses_libedit:
1708 1726 for rlcommand in self.readline_parse_and_bind:
1709 1727 #print "loading rl:",rlcommand # dbg
1710 1728 readline.parse_and_bind(rlcommand)
1711 1729
1712 1730 # Remove some chars from the delimiters list. If we encounter
1713 1731 # unicode chars, discard them.
1714 1732 delims = readline.get_completer_delims()
1715 1733 if not py3compat.PY3:
1716 1734 delims = delims.encode("ascii", "ignore")
1717 1735 for d in self.readline_remove_delims:
1718 1736 delims = delims.replace(d, "")
1719 1737 delims = delims.replace(ESC_MAGIC, '')
1720 1738 readline.set_completer_delims(delims)
1721 1739 # otherwise we end up with a monster history after a while:
1722 1740 readline.set_history_length(self.history_length)
1723 1741
1724 1742 self.refill_readline_hist()
1725 1743 self.readline_no_record = ReadlineNoRecord(self)
1726 1744
1727 1745 # Configure auto-indent for all platforms
1728 1746 self.set_autoindent(self.autoindent)
1729 1747
1730 1748 def refill_readline_hist(self):
1731 1749 # Load the last 1000 lines from history
1732 1750 self.readline.clear_history()
1733 1751 stdin_encoding = sys.stdin.encoding or "utf-8"
1734 1752 last_cell = u""
1735 1753 for _, _, cell in self.history_manager.get_tail(1000,
1736 1754 include_latest=True):
1737 1755 # Ignore blank lines and consecutive duplicates
1738 1756 cell = cell.rstrip()
1739 1757 if cell and (cell != last_cell):
1740 1758 if self.multiline_history:
1741 1759 self.readline.add_history(py3compat.unicode_to_str(cell,
1742 1760 stdin_encoding))
1743 1761 else:
1744 1762 for line in cell.splitlines():
1745 1763 self.readline.add_history(py3compat.unicode_to_str(line,
1746 1764 stdin_encoding))
1747 1765 last_cell = cell
1748 1766
1749 1767 def set_next_input(self, s):
1750 1768 """ Sets the 'default' input string for the next command line.
1751 1769
1752 1770 Requires readline.
1753 1771
1754 1772 Example:
1755 1773
1756 1774 [D:\ipython]|1> _ip.set_next_input("Hello Word")
1757 1775 [D:\ipython]|2> Hello Word_ # cursor is here
1758 1776 """
1759 1777 self.rl_next_input = py3compat.cast_bytes_py2(s)
1760 1778
1761 1779 # Maybe move this to the terminal subclass?
1762 1780 def pre_readline(self):
1763 1781 """readline hook to be used at the start of each line.
1764 1782
1765 1783 Currently it handles auto-indent only."""
1766 1784
1767 1785 if self.rl_do_indent:
1768 1786 self.readline.insert_text(self._indent_current_str())
1769 1787 if self.rl_next_input is not None:
1770 1788 self.readline.insert_text(self.rl_next_input)
1771 1789 self.rl_next_input = None
1772 1790
1773 1791 def _indent_current_str(self):
1774 1792 """return the current level of indentation as a string"""
1775 1793 return self.input_splitter.indent_spaces * ' '
1776 1794
1777 1795 #-------------------------------------------------------------------------
1778 1796 # Things related to text completion
1779 1797 #-------------------------------------------------------------------------
1780 1798
1781 1799 def init_completer(self):
1782 1800 """Initialize the completion machinery.
1783 1801
1784 1802 This creates completion machinery that can be used by client code,
1785 1803 either interactively in-process (typically triggered by the readline
1786 1804 library), programatically (such as in test suites) or out-of-prcess
1787 1805 (typically over the network by remote frontends).
1788 1806 """
1789 1807 from IPython.core.completer import IPCompleter
1790 1808 from IPython.core.completerlib import (module_completer,
1791 1809 magic_run_completer, cd_completer)
1792 1810
1793 1811 self.Completer = IPCompleter(shell=self,
1794 1812 namespace=self.user_ns,
1795 1813 global_namespace=self.user_global_ns,
1796 1814 alias_table=self.alias_manager.alias_table,
1797 1815 use_readline=self.has_readline,
1798 1816 config=self.config,
1799 1817 )
1800 1818 self.configurables.append(self.Completer)
1801 1819
1802 1820 # Add custom completers to the basic ones built into IPCompleter
1803 1821 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1804 1822 self.strdispatchers['complete_command'] = sdisp
1805 1823 self.Completer.custom_completers = sdisp
1806 1824
1807 1825 self.set_hook('complete_command', module_completer, str_key = 'import')
1808 1826 self.set_hook('complete_command', module_completer, str_key = 'from')
1809 1827 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1810 1828 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1811 1829
1812 1830 # Only configure readline if we truly are using readline. IPython can
1813 1831 # do tab-completion over the network, in GUIs, etc, where readline
1814 1832 # itself may be absent
1815 1833 if self.has_readline:
1816 1834 self.set_readline_completer()
1817 1835
1818 1836 def complete(self, text, line=None, cursor_pos=None):
1819 1837 """Return the completed text and a list of completions.
1820 1838
1821 1839 Parameters
1822 1840 ----------
1823 1841
1824 1842 text : string
1825 1843 A string of text to be completed on. It can be given as empty and
1826 1844 instead a line/position pair are given. In this case, the
1827 1845 completer itself will split the line like readline does.
1828 1846
1829 1847 line : string, optional
1830 1848 The complete line that text is part of.
1831 1849
1832 1850 cursor_pos : int, optional
1833 1851 The position of the cursor on the input line.
1834 1852
1835 1853 Returns
1836 1854 -------
1837 1855 text : string
1838 1856 The actual text that was completed.
1839 1857
1840 1858 matches : list
1841 1859 A sorted list with all possible completions.
1842 1860
1843 1861 The optional arguments allow the completion to take more context into
1844 1862 account, and are part of the low-level completion API.
1845 1863
1846 1864 This is a wrapper around the completion mechanism, similar to what
1847 1865 readline does at the command line when the TAB key is hit. By
1848 1866 exposing it as a method, it can be used by other non-readline
1849 1867 environments (such as GUIs) for text completion.
1850 1868
1851 1869 Simple usage example:
1852 1870
1853 1871 In [1]: x = 'hello'
1854 1872
1855 1873 In [2]: _ip.complete('x.l')
1856 1874 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
1857 1875 """
1858 1876
1859 1877 # Inject names into __builtin__ so we can complete on the added names.
1860 1878 with self.builtin_trap:
1861 1879 return self.Completer.complete(text, line, cursor_pos)
1862 1880
1863 1881 def set_custom_completer(self, completer, pos=0):
1864 1882 """Adds a new custom completer function.
1865 1883
1866 1884 The position argument (defaults to 0) is the index in the completers
1867 1885 list where you want the completer to be inserted."""
1868 1886
1869 1887 newcomp = types.MethodType(completer,self.Completer)
1870 1888 self.Completer.matchers.insert(pos,newcomp)
1871 1889
1872 1890 def set_readline_completer(self):
1873 1891 """Reset readline's completer to be our own."""
1874 1892 self.readline.set_completer(self.Completer.rlcomplete)
1875 1893
1876 1894 def set_completer_frame(self, frame=None):
1877 1895 """Set the frame of the completer."""
1878 1896 if frame:
1879 1897 self.Completer.namespace = frame.f_locals
1880 1898 self.Completer.global_namespace = frame.f_globals
1881 1899 else:
1882 1900 self.Completer.namespace = self.user_ns
1883 1901 self.Completer.global_namespace = self.user_global_ns
1884 1902
1885 1903 #-------------------------------------------------------------------------
1886 1904 # Things related to magics
1887 1905 #-------------------------------------------------------------------------
1888 1906
1889 1907 def init_magics(self):
1890 1908 # FIXME: Move the color initialization to the DisplayHook, which
1891 1909 # should be split into a prompt manager and displayhook. We probably
1892 1910 # even need a centralize colors management object.
1893 1911 self.magic_colors(self.colors)
1894 1912 # History was moved to a separate module
1895 1913 from . import history
1896 1914 history.init_ipython(self)
1897 1915
1898 1916 def magic(self, arg_s, next_input=None):
1899 1917 """Call a magic function by name.
1900 1918
1901 1919 Input: a string containing the name of the magic function to call and
1902 1920 any additional arguments to be passed to the magic.
1903 1921
1904 1922 magic('name -opt foo bar') is equivalent to typing at the ipython
1905 1923 prompt:
1906 1924
1907 1925 In[1]: %name -opt foo bar
1908 1926
1909 1927 To call a magic without arguments, simply use magic('name').
1910 1928
1911 1929 This provides a proper Python function to call IPython's magics in any
1912 1930 valid Python code you can type at the interpreter, including loops and
1913 1931 compound statements.
1914 1932 """
1915 1933 # Allow setting the next input - this is used if the user does `a=abs?`.
1916 1934 # We do this first so that magic functions can override it.
1917 1935 if next_input:
1918 1936 self.set_next_input(next_input)
1919 1937
1920 1938 args = arg_s.split(' ',1)
1921 1939 magic_name = args[0]
1922 1940 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
1923 1941
1924 1942 try:
1925 1943 magic_args = args[1]
1926 1944 except IndexError:
1927 1945 magic_args = ''
1928 1946 fn = getattr(self,'magic_'+magic_name,None)
1929 1947 if fn is None:
1930 1948 error("Magic function `%s` not found." % magic_name)
1931 1949 else:
1932 1950 magic_args = self.var_expand(magic_args,1)
1933 1951 # Grab local namespace if we need it:
1934 1952 if getattr(fn, "needs_local_scope", False):
1935 1953 self._magic_locals = sys._getframe(1).f_locals
1936 1954 with self.builtin_trap:
1937 1955 result = fn(magic_args)
1938 1956 # Ensure we're not keeping object references around:
1939 1957 self._magic_locals = {}
1940 1958 return result
1941 1959
1942 1960 def define_magic(self, magicname, func):
1943 1961 """Expose own function as magic function for ipython
1944 1962
1945 1963 Example::
1946 1964
1947 1965 def foo_impl(self,parameter_s=''):
1948 1966 'My very own magic!. (Use docstrings, IPython reads them).'
1949 1967 print 'Magic function. Passed parameter is between < >:'
1950 1968 print '<%s>' % parameter_s
1951 1969 print 'The self object is:', self
1952 1970
1953 1971 ip.define_magic('foo',foo_impl)
1954 1972 """
1955 1973 im = types.MethodType(func,self)
1956 1974 old = getattr(self, "magic_" + magicname, None)
1957 1975 setattr(self, "magic_" + magicname, im)
1958 1976 return old
1959 1977
1960 1978 #-------------------------------------------------------------------------
1961 1979 # Things related to macros
1962 1980 #-------------------------------------------------------------------------
1963 1981
1964 1982 def define_macro(self, name, themacro):
1965 1983 """Define a new macro
1966 1984
1967 1985 Parameters
1968 1986 ----------
1969 1987 name : str
1970 1988 The name of the macro.
1971 1989 themacro : str or Macro
1972 1990 The action to do upon invoking the macro. If a string, a new
1973 1991 Macro object is created by passing the string to it.
1974 1992 """
1975 1993
1976 1994 from IPython.core import macro
1977 1995
1978 1996 if isinstance(themacro, basestring):
1979 1997 themacro = macro.Macro(themacro)
1980 1998 if not isinstance(themacro, macro.Macro):
1981 1999 raise ValueError('A macro must be a string or a Macro instance.')
1982 2000 self.user_ns[name] = themacro
1983 2001
1984 2002 #-------------------------------------------------------------------------
1985 2003 # Things related to the running of system commands
1986 2004 #-------------------------------------------------------------------------
1987 2005
1988 2006 def system_piped(self, cmd):
1989 2007 """Call the given cmd in a subprocess, piping stdout/err
1990 2008
1991 2009 Parameters
1992 2010 ----------
1993 2011 cmd : str
1994 2012 Command to execute (can not end in '&', as background processes are
1995 2013 not supported. Should not be a command that expects input
1996 2014 other than simple text.
1997 2015 """
1998 2016 if cmd.rstrip().endswith('&'):
1999 2017 # this is *far* from a rigorous test
2000 2018 # We do not support backgrounding processes because we either use
2001 2019 # pexpect or pipes to read from. Users can always just call
2002 2020 # os.system() or use ip.system=ip.system_raw
2003 2021 # if they really want a background process.
2004 2022 raise OSError("Background processes not supported.")
2005 2023
2006 2024 # we explicitly do NOT return the subprocess status code, because
2007 2025 # a non-None value would trigger :func:`sys.displayhook` calls.
2008 2026 # Instead, we store the exit_code in user_ns.
2009 2027 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=2))
2010 2028
2011 2029 def system_raw(self, cmd):
2012 2030 """Call the given cmd in a subprocess using os.system
2013 2031
2014 2032 Parameters
2015 2033 ----------
2016 2034 cmd : str
2017 2035 Command to execute.
2018 2036 """
2019 2037 cmd = self.var_expand(cmd, depth=2)
2020 2038 # protect os.system from UNC paths on Windows, which it can't handle:
2021 2039 if sys.platform == 'win32':
2022 2040 from IPython.utils._process_win32 import AvoidUNCPath
2023 2041 with AvoidUNCPath() as path:
2024 2042 if path is not None:
2025 2043 cmd = '"pushd %s &&"%s' % (path, cmd)
2026 2044 cmd = py3compat.unicode_to_str(cmd)
2027 2045 ec = os.system(cmd)
2028 2046 else:
2029 2047 cmd = py3compat.unicode_to_str(cmd)
2030 2048 ec = os.system(cmd)
2031 2049
2032 2050 # We explicitly do NOT return the subprocess status code, because
2033 2051 # a non-None value would trigger :func:`sys.displayhook` calls.
2034 2052 # Instead, we store the exit_code in user_ns.
2035 2053 self.user_ns['_exit_code'] = ec
2036 2054
2037 2055 # use piped system by default, because it is better behaved
2038 2056 system = system_piped
2039 2057
2040 2058 def getoutput(self, cmd, split=True):
2041 2059 """Get output (possibly including stderr) from a subprocess.
2042 2060
2043 2061 Parameters
2044 2062 ----------
2045 2063 cmd : str
2046 2064 Command to execute (can not end in '&', as background processes are
2047 2065 not supported.
2048 2066 split : bool, optional
2049 2067
2050 2068 If True, split the output into an IPython SList. Otherwise, an
2051 2069 IPython LSString is returned. These are objects similar to normal
2052 2070 lists and strings, with a few convenience attributes for easier
2053 2071 manipulation of line-based output. You can use '?' on them for
2054 2072 details.
2055 2073 """
2056 2074 if cmd.rstrip().endswith('&'):
2057 2075 # this is *far* from a rigorous test
2058 2076 raise OSError("Background processes not supported.")
2059 2077 out = getoutput(self.var_expand(cmd, depth=2))
2060 2078 if split:
2061 2079 out = SList(out.splitlines())
2062 2080 else:
2063 2081 out = LSString(out)
2064 2082 return out
2065 2083
2066 2084 #-------------------------------------------------------------------------
2067 2085 # Things related to aliases
2068 2086 #-------------------------------------------------------------------------
2069 2087
2070 2088 def init_alias(self):
2071 2089 self.alias_manager = AliasManager(shell=self, config=self.config)
2072 2090 self.configurables.append(self.alias_manager)
2073 2091 self.ns_table['alias'] = self.alias_manager.alias_table,
2074 2092
2075 2093 #-------------------------------------------------------------------------
2076 2094 # Things related to extensions and plugins
2077 2095 #-------------------------------------------------------------------------
2078 2096
2079 2097 def init_extension_manager(self):
2080 2098 self.extension_manager = ExtensionManager(shell=self, config=self.config)
2081 2099 self.configurables.append(self.extension_manager)
2082 2100
2083 2101 def init_plugin_manager(self):
2084 2102 self.plugin_manager = PluginManager(config=self.config)
2085 2103 self.configurables.append(self.plugin_manager)
2086 2104
2087 2105
2088 2106 #-------------------------------------------------------------------------
2089 2107 # Things related to payloads
2090 2108 #-------------------------------------------------------------------------
2091 2109
2092 2110 def init_payload(self):
2093 2111 self.payload_manager = PayloadManager(config=self.config)
2094 2112 self.configurables.append(self.payload_manager)
2095 2113
2096 2114 #-------------------------------------------------------------------------
2097 2115 # Things related to the prefilter
2098 2116 #-------------------------------------------------------------------------
2099 2117
2100 2118 def init_prefilter(self):
2101 2119 self.prefilter_manager = PrefilterManager(shell=self, config=self.config)
2102 2120 self.configurables.append(self.prefilter_manager)
2103 2121 # Ultimately this will be refactored in the new interpreter code, but
2104 2122 # for now, we should expose the main prefilter method (there's legacy
2105 2123 # code out there that may rely on this).
2106 2124 self.prefilter = self.prefilter_manager.prefilter_lines
2107 2125
2108 2126 def auto_rewrite_input(self, cmd):
2109 2127 """Print to the screen the rewritten form of the user's command.
2110 2128
2111 2129 This shows visual feedback by rewriting input lines that cause
2112 2130 automatic calling to kick in, like::
2113 2131
2114 2132 /f x
2115 2133
2116 2134 into::
2117 2135
2118 2136 ------> f(x)
2119 2137
2120 2138 after the user's input prompt. This helps the user understand that the
2121 2139 input line was transformed automatically by IPython.
2122 2140 """
2123 2141 rw = self.displayhook.prompt1.auto_rewrite() + cmd
2124 2142
2125 2143 try:
2126 2144 # plain ascii works better w/ pyreadline, on some machines, so
2127 2145 # we use it and only print uncolored rewrite if we have unicode
2128 2146 rw = str(rw)
2129 2147 print >> io.stdout, rw
2130 2148 except UnicodeEncodeError:
2131 2149 print "------> " + cmd
2132 2150
2133 2151 #-------------------------------------------------------------------------
2134 2152 # Things related to extracting values/expressions from kernel and user_ns
2135 2153 #-------------------------------------------------------------------------
2136 2154
2137 2155 def _simple_error(self):
2138 2156 etype, value = sys.exc_info()[:2]
2139 2157 return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
2140 2158
2141 2159 def user_variables(self, names):
2142 2160 """Get a list of variable names from the user's namespace.
2143 2161
2144 2162 Parameters
2145 2163 ----------
2146 2164 names : list of strings
2147 2165 A list of names of variables to be read from the user namespace.
2148 2166
2149 2167 Returns
2150 2168 -------
2151 2169 A dict, keyed by the input names and with the repr() of each value.
2152 2170 """
2153 2171 out = {}
2154 2172 user_ns = self.user_ns
2155 2173 for varname in names:
2156 2174 try:
2157 2175 value = repr(user_ns[varname])
2158 2176 except:
2159 2177 value = self._simple_error()
2160 2178 out[varname] = value
2161 2179 return out
2162 2180
2163 2181 def user_expressions(self, expressions):
2164 2182 """Evaluate a dict of expressions in the user's namespace.
2165 2183
2166 2184 Parameters
2167 2185 ----------
2168 2186 expressions : dict
2169 2187 A dict with string keys and string values. The expression values
2170 2188 should be valid Python expressions, each of which will be evaluated
2171 2189 in the user namespace.
2172 2190
2173 2191 Returns
2174 2192 -------
2175 2193 A dict, keyed like the input expressions dict, with the repr() of each
2176 2194 value.
2177 2195 """
2178 2196 out = {}
2179 2197 user_ns = self.user_ns
2180 2198 global_ns = self.user_global_ns
2181 2199 for key, expr in expressions.iteritems():
2182 2200 try:
2183 2201 value = repr(eval(expr, global_ns, user_ns))
2184 2202 except:
2185 2203 value = self._simple_error()
2186 2204 out[key] = value
2187 2205 return out
2188 2206
2189 2207 #-------------------------------------------------------------------------
2190 2208 # Things related to the running of code
2191 2209 #-------------------------------------------------------------------------
2192 2210
2193 2211 def ex(self, cmd):
2194 2212 """Execute a normal python statement in user namespace."""
2195 2213 with self.builtin_trap:
2196 2214 exec cmd in self.user_global_ns, self.user_ns
2197 2215
2198 2216 def ev(self, expr):
2199 2217 """Evaluate python expression expr in user namespace.
2200 2218
2201 2219 Returns the result of evaluation
2202 2220 """
2203 2221 with self.builtin_trap:
2204 2222 return eval(expr, self.user_global_ns, self.user_ns)
2205 2223
2206 2224 def safe_execfile(self, fname, *where, **kw):
2207 2225 """A safe version of the builtin execfile().
2208 2226
2209 2227 This version will never throw an exception, but instead print
2210 2228 helpful error messages to the screen. This only works on pure
2211 2229 Python files with the .py extension.
2212 2230
2213 2231 Parameters
2214 2232 ----------
2215 2233 fname : string
2216 2234 The name of the file to be executed.
2217 2235 where : tuple
2218 2236 One or two namespaces, passed to execfile() as (globals,locals).
2219 2237 If only one is given, it is passed as both.
2220 2238 exit_ignore : bool (False)
2221 2239 If True, then silence SystemExit for non-zero status (it is always
2222 2240 silenced for zero status, as it is so common).
2223 2241 raise_exceptions : bool (False)
2224 2242 If True raise exceptions everywhere. Meant for testing.
2225 2243
2226 2244 """
2227 2245 kw.setdefault('exit_ignore', False)
2228 2246 kw.setdefault('raise_exceptions', False)
2229 2247
2230 2248 fname = os.path.abspath(os.path.expanduser(fname))
2231 2249
2232 2250 # Make sure we can open the file
2233 2251 try:
2234 2252 with open(fname) as thefile:
2235 2253 pass
2236 2254 except:
2237 2255 warn('Could not open file <%s> for safe execution.' % fname)
2238 2256 return
2239 2257
2240 2258 # Find things also in current directory. This is needed to mimic the
2241 2259 # behavior of running a script from the system command line, where
2242 2260 # Python inserts the script's directory into sys.path
2243 2261 dname = os.path.dirname(fname)
2244 2262
2245 2263 with prepended_to_syspath(dname):
2246 2264 try:
2247 2265 py3compat.execfile(fname,*where)
2248 2266 except SystemExit, status:
2249 2267 # If the call was made with 0 or None exit status (sys.exit(0)
2250 2268 # or sys.exit() ), don't bother showing a traceback, as both of
2251 2269 # these are considered normal by the OS:
2252 2270 # > python -c'import sys;sys.exit(0)'; echo $?
2253 2271 # 0
2254 2272 # > python -c'import sys;sys.exit()'; echo $?
2255 2273 # 0
2256 2274 # For other exit status, we show the exception unless
2257 2275 # explicitly silenced, but only in short form.
2258 2276 if kw['raise_exceptions']:
2259 2277 raise
2260 2278 if status.code not in (0, None) and not kw['exit_ignore']:
2261 2279 self.showtraceback(exception_only=True)
2262 2280 except:
2263 2281 if kw['raise_exceptions']:
2264 2282 raise
2265 2283 self.showtraceback()
2266 2284
2267 2285 def safe_execfile_ipy(self, fname):
2268 2286 """Like safe_execfile, but for .ipy files with IPython syntax.
2269 2287
2270 2288 Parameters
2271 2289 ----------
2272 2290 fname : str
2273 2291 The name of the file to execute. The filename must have a
2274 2292 .ipy extension.
2275 2293 """
2276 2294 fname = os.path.abspath(os.path.expanduser(fname))
2277 2295
2278 2296 # Make sure we can open the file
2279 2297 try:
2280 2298 with open(fname) as thefile:
2281 2299 pass
2282 2300 except:
2283 2301 warn('Could not open file <%s> for safe execution.' % fname)
2284 2302 return
2285 2303
2286 2304 # Find things also in current directory. This is needed to mimic the
2287 2305 # behavior of running a script from the system command line, where
2288 2306 # Python inserts the script's directory into sys.path
2289 2307 dname = os.path.dirname(fname)
2290 2308
2291 2309 with prepended_to_syspath(dname):
2292 2310 try:
2293 2311 with open(fname) as thefile:
2294 2312 # self.run_cell currently captures all exceptions
2295 2313 # raised in user code. It would be nice if there were
2296 2314 # versions of runlines, execfile that did raise, so
2297 2315 # we could catch the errors.
2298 2316 self.run_cell(thefile.read(), store_history=False)
2299 2317 except:
2300 2318 self.showtraceback()
2301 2319 warn('Unknown failure executing file: <%s>' % fname)
2302 2320
2303 2321 def run_cell(self, raw_cell, store_history=False):
2304 2322 """Run a complete IPython cell.
2305 2323
2306 2324 Parameters
2307 2325 ----------
2308 2326 raw_cell : str
2309 2327 The code (including IPython code such as %magic functions) to run.
2310 2328 store_history : bool
2311 2329 If True, the raw and translated cell will be stored in IPython's
2312 2330 history. For user code calling back into IPython's machinery, this
2313 2331 should be set to False.
2314 2332 """
2315 2333 if (not raw_cell) or raw_cell.isspace():
2316 2334 return
2317 2335
2318 2336 for line in raw_cell.splitlines():
2319 2337 self.input_splitter.push(line)
2320 2338 cell = self.input_splitter.source_reset()
2321 2339
2322 2340 with self.builtin_trap:
2323 2341 prefilter_failed = False
2324 2342 if len(cell.splitlines()) == 1:
2325 2343 try:
2326 2344 # use prefilter_lines to handle trailing newlines
2327 2345 # restore trailing newline for ast.parse
2328 2346 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2329 2347 except AliasError as e:
2330 2348 error(e)
2331 2349 prefilter_failed = True
2332 2350 except Exception:
2333 2351 # don't allow prefilter errors to crash IPython
2334 2352 self.showtraceback()
2335 2353 prefilter_failed = True
2336 2354
2337 2355 # Store raw and processed history
2338 2356 if store_history:
2339 2357 self.history_manager.store_inputs(self.execution_count,
2340 2358 cell, raw_cell)
2341 2359
2342 2360 self.logger.log(cell, raw_cell)
2343 2361
2344 2362 if not prefilter_failed:
2345 2363 # don't run if prefilter failed
2346 2364 cell_name = self.compile.cache(cell, self.execution_count)
2347 2365
2348 2366 with self.display_trap:
2349 2367 try:
2350 2368 code_ast = self.compile.ast_parse(cell, filename=cell_name)
2351 2369 except IndentationError:
2352 2370 self.showindentationerror()
2353 2371 self.execution_count += 1
2354 2372 return None
2355 2373 except (OverflowError, SyntaxError, ValueError, TypeError,
2356 2374 MemoryError):
2357 2375 self.showsyntaxerror()
2358 2376 self.execution_count += 1
2359 2377 return None
2360 2378
2361 2379 self.run_ast_nodes(code_ast.body, cell_name,
2362 2380 interactivity="last_expr")
2363 2381
2364 2382 # Execute any registered post-execution functions.
2365 2383 for func, status in self._post_execute.iteritems():
2366 2384 if not status:
2367 2385 continue
2368 2386 try:
2369 2387 func()
2370 2388 except KeyboardInterrupt:
2371 2389 print >> io.stderr, "\nKeyboardInterrupt"
2372 2390 except Exception:
2373 2391 print >> io.stderr, "Disabling failed post-execution function: %s" % func
2374 2392 self.showtraceback()
2375 2393 # Deactivate failing function
2376 2394 self._post_execute[func] = False
2377 2395
2378 2396 if store_history:
2379 2397 # Write output to the database. Does nothing unless
2380 2398 # history output logging is enabled.
2381 2399 self.history_manager.store_output(self.execution_count)
2382 2400 # Each cell is a *single* input, regardless of how many lines it has
2383 2401 self.execution_count += 1
2384 2402
2385 2403 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr'):
2386 2404 """Run a sequence of AST nodes. The execution mode depends on the
2387 2405 interactivity parameter.
2388 2406
2389 2407 Parameters
2390 2408 ----------
2391 2409 nodelist : list
2392 2410 A sequence of AST nodes to run.
2393 2411 cell_name : str
2394 2412 Will be passed to the compiler as the filename of the cell. Typically
2395 2413 the value returned by ip.compile.cache(cell).
2396 2414 interactivity : str
2397 2415 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2398 2416 run interactively (displaying output from expressions). 'last_expr'
2399 2417 will run the last node interactively only if it is an expression (i.e.
2400 2418 expressions in loops or other blocks are not displayed. Other values
2401 2419 for this parameter will raise a ValueError.
2402 2420 """
2403 2421 if not nodelist:
2404 2422 return
2405 2423
2406 2424 if interactivity == 'last_expr':
2407 2425 if isinstance(nodelist[-1], ast.Expr):
2408 2426 interactivity = "last"
2409 2427 else:
2410 2428 interactivity = "none"
2411 2429
2412 2430 if interactivity == 'none':
2413 2431 to_run_exec, to_run_interactive = nodelist, []
2414 2432 elif interactivity == 'last':
2415 2433 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2416 2434 elif interactivity == 'all':
2417 2435 to_run_exec, to_run_interactive = [], nodelist
2418 2436 else:
2419 2437 raise ValueError("Interactivity was %r" % interactivity)
2420 2438
2421 2439 exec_count = self.execution_count
2422 2440
2423 2441 try:
2424 2442 for i, node in enumerate(to_run_exec):
2425 2443 mod = ast.Module([node])
2426 2444 code = self.compile(mod, cell_name, "exec")
2427 2445 if self.run_code(code):
2428 2446 return True
2429 2447
2430 2448 for i, node in enumerate(to_run_interactive):
2431 2449 mod = ast.Interactive([node])
2432 2450 code = self.compile(mod, cell_name, "single")
2433 2451 if self.run_code(code):
2434 2452 return True
2435 2453 except:
2436 2454 # It's possible to have exceptions raised here, typically by
2437 2455 # compilation of odd code (such as a naked 'return' outside a
2438 2456 # function) that did parse but isn't valid. Typically the exception
2439 2457 # is a SyntaxError, but it's safest just to catch anything and show
2440 2458 # the user a traceback.
2441 2459
2442 2460 # We do only one try/except outside the loop to minimize the impact
2443 2461 # on runtime, and also because if any node in the node list is
2444 2462 # broken, we should stop execution completely.
2445 2463 self.showtraceback()
2446 2464
2447 2465 return False
2448 2466
2449 2467 def run_code(self, code_obj):
2450 2468 """Execute a code object.
2451 2469
2452 2470 When an exception occurs, self.showtraceback() is called to display a
2453 2471 traceback.
2454 2472
2455 2473 Parameters
2456 2474 ----------
2457 2475 code_obj : code object
2458 2476 A compiled code object, to be executed
2459 2477 post_execute : bool [default: True]
2460 2478 whether to call post_execute hooks after this particular execution.
2461 2479
2462 2480 Returns
2463 2481 -------
2464 2482 False : successful execution.
2465 2483 True : an error occurred.
2466 2484 """
2467 2485
2468 2486 # Set our own excepthook in case the user code tries to call it
2469 2487 # directly, so that the IPython crash handler doesn't get triggered
2470 2488 old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
2471 2489
2472 2490 # we save the original sys.excepthook in the instance, in case config
2473 2491 # code (such as magics) needs access to it.
2474 2492 self.sys_excepthook = old_excepthook
2475 2493 outflag = 1 # happens in more places, so it's easier as default
2476 2494 try:
2477 2495 try:
2478 2496 self.hooks.pre_run_code_hook()
2479 2497 #rprint('Running code', repr(code_obj)) # dbg
2480 2498 exec code_obj in self.user_global_ns, self.user_ns
2481 2499 finally:
2482 2500 # Reset our crash handler in place
2483 2501 sys.excepthook = old_excepthook
2484 2502 except SystemExit:
2485 2503 self.showtraceback(exception_only=True)
2486 2504 warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1)
2487 2505 except self.custom_exceptions:
2488 2506 etype,value,tb = sys.exc_info()
2489 2507 self.CustomTB(etype,value,tb)
2490 2508 except:
2491 2509 self.showtraceback()
2492 2510 else:
2493 2511 outflag = 0
2494 2512 if softspace(sys.stdout, 0):
2495 2513 print
2496 2514
2497 2515 return outflag
2498 2516
2499 2517 # For backwards compatibility
2500 2518 runcode = run_code
2501 2519
2502 2520 #-------------------------------------------------------------------------
2503 2521 # Things related to GUI support and pylab
2504 2522 #-------------------------------------------------------------------------
2505 2523
2506 2524 def enable_pylab(self, gui=None, import_all=True):
2507 2525 raise NotImplementedError('Implement enable_pylab in a subclass')
2508 2526
2509 2527 #-------------------------------------------------------------------------
2510 2528 # Utilities
2511 2529 #-------------------------------------------------------------------------
2512 2530
2513 2531 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
2514 2532 """Expand python variables in a string.
2515 2533
2516 2534 The depth argument indicates how many frames above the caller should
2517 2535 be walked to look for the local namespace where to expand variables.
2518 2536
2519 2537 The global namespace for expansion is always the user's interactive
2520 2538 namespace.
2521 2539 """
2522 2540 ns = self.user_ns.copy()
2523 2541 ns.update(sys._getframe(depth+1).f_locals)
2524 2542 ns.pop('self', None)
2525 2543 return formatter.format(cmd, **ns)
2526 2544
2527 2545 def mktempfile(self, data=None, prefix='ipython_edit_'):
2528 2546 """Make a new tempfile and return its filename.
2529 2547
2530 2548 This makes a call to tempfile.mktemp, but it registers the created
2531 2549 filename internally so ipython cleans it up at exit time.
2532 2550
2533 2551 Optional inputs:
2534 2552
2535 2553 - data(None): if data is given, it gets written out to the temp file
2536 2554 immediately, and the file is closed again."""
2537 2555
2538 2556 filename = tempfile.mktemp('.py', prefix)
2539 2557 self.tempfiles.append(filename)
2540 2558
2541 2559 if data:
2542 2560 tmp_file = open(filename,'w')
2543 2561 tmp_file.write(data)
2544 2562 tmp_file.close()
2545 2563 return filename
2546 2564
2547 2565 # TODO: This should be removed when Term is refactored.
2548 2566 def write(self,data):
2549 2567 """Write a string to the default output"""
2550 2568 io.stdout.write(data)
2551 2569
2552 2570 # TODO: This should be removed when Term is refactored.
2553 2571 def write_err(self,data):
2554 2572 """Write a string to the default error output"""
2555 2573 io.stderr.write(data)
2556 2574
2557 2575 def ask_yes_no(self, prompt, default=None):
2558 2576 if self.quiet:
2559 2577 return True
2560 2578 return ask_yes_no(prompt,default)
2561 2579
2562 2580 def show_usage(self):
2563 2581 """Show a usage message"""
2564 2582 page.page(IPython.core.usage.interactive_usage)
2565 2583
2566 2584 def find_user_code(self, target, raw=True):
2567 2585 """Get a code string from history, file, or a string or macro.
2568 2586
2569 2587 This is mainly used by magic functions.
2570 2588
2571 2589 Parameters
2572 2590 ----------
2573 2591 target : str
2574 2592 A string specifying code to retrieve. This will be tried respectively
2575 2593 as: ranges of input history (see %history for syntax), a filename, or
2576 2594 an expression evaluating to a string or Macro in the user namespace.
2577 2595 raw : bool
2578 2596 If true (default), retrieve raw history. Has no effect on the other
2579 2597 retrieval mechanisms.
2580 2598
2581 2599 Returns
2582 2600 -------
2583 2601 A string of code.
2584 2602
2585 2603 ValueError is raised if nothing is found, and TypeError if it evaluates
2586 2604 to an object of another type. In each case, .args[0] is a printable
2587 2605 message.
2588 2606 """
2589 2607 code = self.extract_input_lines(target, raw=raw) # Grab history
2590 2608 if code:
2591 2609 return code
2592 2610 if os.path.isfile(target): # Read file
2593 2611 return open(target, "r").read()
2594 2612
2595 2613 try: # User namespace
2596 2614 codeobj = eval(target, self.user_ns)
2597 2615 except Exception:
2598 2616 raise ValueError(("'%s' was not found in history, as a file, nor in"
2599 2617 " the user namespace.") % target)
2600 2618 if isinstance(codeobj, basestring):
2601 2619 return codeobj
2602 2620 elif isinstance(codeobj, Macro):
2603 2621 return codeobj.value
2604 2622
2605 2623 raise TypeError("%s is neither a string nor a macro." % target,
2606 2624 codeobj)
2607 2625
2608 2626 #-------------------------------------------------------------------------
2609 2627 # Things related to IPython exiting
2610 2628 #-------------------------------------------------------------------------
2611 2629 def atexit_operations(self):
2612 2630 """This will be executed at the time of exit.
2613 2631
2614 2632 Cleanup operations and saving of persistent data that is done
2615 2633 unconditionally by IPython should be performed here.
2616 2634
2617 2635 For things that may depend on startup flags or platform specifics (such
2618 2636 as having readline or not), register a separate atexit function in the
2619 2637 code that has the appropriate information, rather than trying to
2620 2638 clutter
2621 2639 """
2622 2640 # Close the history session (this stores the end time and line count)
2623 2641 # this must be *before* the tempfile cleanup, in case of temporary
2624 2642 # history db
2625 2643 self.history_manager.end_session()
2626 2644
2627 2645 # Cleanup all tempfiles left around
2628 2646 for tfile in self.tempfiles:
2629 2647 try:
2630 2648 os.unlink(tfile)
2631 2649 except OSError:
2632 2650 pass
2633 2651
2634 2652 # Clear all user namespaces to release all references cleanly.
2635 2653 self.reset(new_session=False)
2636 2654
2637 2655 # Run user hooks
2638 2656 self.hooks.shutdown_hook()
2639 2657
2640 2658 def cleanup(self):
2641 2659 self.restore_sys_module_state()
2642 2660
2643 2661
2644 2662 class InteractiveShellABC(object):
2645 2663 """An abstract base class for InteractiveShell."""
2646 2664 __metaclass__ = abc.ABCMeta
2647 2665
2648 2666 InteractiveShellABC.register(InteractiveShell)
@@ -1,240 +1,273 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the key interactiveshell module.
3 3
4 4 Historically the main classes in interactiveshell have been under-tested. This
5 5 module should grow as many single-method tests as possible to trap many of the
6 6 recurring bugs we seem to encounter with high-level interaction.
7 7
8 8 Authors
9 9 -------
10 10 * Fernando Perez
11 11 """
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2011 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22 # stdlib
23 23 import os
24 24 import shutil
25 25 import tempfile
26 26 import unittest
27 27 from os.path import join
28 import sys
28 29 from StringIO import StringIO
29 30
30 31 from IPython.testing import decorators as dec
31 32 from IPython.utils import io
32 33
33 34 #-----------------------------------------------------------------------------
34 35 # Tests
35 36 #-----------------------------------------------------------------------------
36 37
37 38 class InteractiveShellTestCase(unittest.TestCase):
38 39 def test_naked_string_cells(self):
39 40 """Test that cells with only naked strings are fully executed"""
40 41 ip = get_ipython()
41 42 # First, single-line inputs
42 43 ip.run_cell('"a"\n')
43 44 self.assertEquals(ip.user_ns['_'], 'a')
44 45 # And also multi-line cells
45 46 ip.run_cell('"""a\nb"""\n')
46 47 self.assertEquals(ip.user_ns['_'], 'a\nb')
47 48
48 49 def test_run_empty_cell(self):
49 50 """Just make sure we don't get a horrible error with a blank
50 51 cell of input. Yes, I did overlook that."""
51 52 ip = get_ipython()
52 53 old_xc = ip.execution_count
53 54 ip.run_cell('')
54 55 self.assertEquals(ip.execution_count, old_xc)
55 56
56 57 def test_run_cell_multiline(self):
57 58 """Multi-block, multi-line cells must execute correctly.
58 59 """
59 60 ip = get_ipython()
60 61 src = '\n'.join(["x=1",
61 62 "y=2",
62 63 "if 1:",
63 64 " x += 1",
64 65 " y += 1",])
65 66 ip.run_cell(src)
66 67 self.assertEquals(ip.user_ns['x'], 2)
67 68 self.assertEquals(ip.user_ns['y'], 3)
68 69
69 70 def test_multiline_string_cells(self):
70 71 "Code sprinkled with multiline strings should execute (GH-306)"
71 72 ip = get_ipython()
72 73 ip.run_cell('tmp=0')
73 74 self.assertEquals(ip.user_ns['tmp'], 0)
74 75 ip.run_cell('tmp=1;"""a\nb"""\n')
75 76 self.assertEquals(ip.user_ns['tmp'], 1)
76 77
77 78 def test_dont_cache_with_semicolon(self):
78 79 "Ending a line with semicolon should not cache the returned object (GH-307)"
79 80 ip = get_ipython()
80 81 oldlen = len(ip.user_ns['Out'])
81 82 a = ip.run_cell('1;', store_history=True)
82 83 newlen = len(ip.user_ns['Out'])
83 84 self.assertEquals(oldlen, newlen)
84 85 #also test the default caching behavior
85 86 ip.run_cell('1', store_history=True)
86 87 newlen = len(ip.user_ns['Out'])
87 88 self.assertEquals(oldlen+1, newlen)
88 89
89 90 def test_In_variable(self):
90 91 "Verify that In variable grows with user input (GH-284)"
91 92 ip = get_ipython()
92 93 oldlen = len(ip.user_ns['In'])
93 94 ip.run_cell('1;', store_history=True)
94 95 newlen = len(ip.user_ns['In'])
95 96 self.assertEquals(oldlen+1, newlen)
96 97 self.assertEquals(ip.user_ns['In'][-1],'1;')
97 98
98 99 def test_magic_names_in_string(self):
99 100 ip = get_ipython()
100 101 ip.run_cell('a = """\n%exit\n"""')
101 102 self.assertEquals(ip.user_ns['a'], '\n%exit\n')
102 103
103 104 def test_alias_crash(self):
104 105 """Errors in prefilter can't crash IPython"""
105 106 ip = get_ipython()
106 107 ip.run_cell('%alias parts echo first %s second %s')
107 108 # capture stderr:
108 109 save_err = io.stderr
109 110 io.stderr = StringIO()
110 111 ip.run_cell('parts 1')
111 112 err = io.stderr.getvalue()
112 113 io.stderr = save_err
113 114 self.assertEquals(err.split(':')[0], 'ERROR')
114 115
115 116 def test_trailing_newline(self):
116 117 """test that running !(command) does not raise a SyntaxError"""
117 118 ip = get_ipython()
118 119 ip.run_cell('!(true)\n', False)
119 120 ip.run_cell('!(true)\n\n\n', False)
120 121
121 122 def test_gh_597(self):
122 123 """Pretty-printing lists of objects with non-ascii reprs may cause
123 124 problems."""
124 125 class Spam(object):
125 126 def __repr__(self):
126 127 return "\xe9"*50
127 128 import IPython.core.formatters
128 129 f = IPython.core.formatters.PlainTextFormatter()
129 130 f([Spam(),Spam()])
130 131
132
131 133 def test_future_flags(self):
132 134 """Check that future flags are used for parsing code (gh-777)"""
133 135 ip = get_ipython()
134 136 ip.run_cell('from __future__ import print_function')
135 137 try:
136 138 ip.run_cell('prfunc_return_val = print(1,2, sep=" ")')
137 139 assert 'prfunc_return_val' in ip.user_ns
138 140 finally:
139 141 # Reset compiler flags so we don't mess up other tests.
140 142 ip.compile.reset_compiler_flags()
141 143
142 144 def test_future_unicode(self):
143 145 """Check that unicode_literals is imported from __future__ (gh #786)"""
144 146 ip = get_ipython()
145 147 try:
146 148 ip.run_cell(u'byte_str = "a"')
147 149 assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default
148 150 ip.run_cell('from __future__ import unicode_literals')
149 151 ip.run_cell(u'unicode_str = "a"')
150 152 assert isinstance(ip.user_ns['unicode_str'], unicode) # strings literals are now unicode
151 153 finally:
152 154 # Reset compiler flags so we don't mess up other tests.
153 155 ip.compile.reset_compiler_flags()
156
157 def test_can_pickle(self):
158 "Can we pickle objects defined interactively (GH-29)"
159 ip = get_ipython()
160 ip.reset()
161 ip.run_cell(("class Mylist(list):\n"
162 " def __init__(self,x=[]):\n"
163 " list.__init__(self,x)"))
164 ip.run_cell("w=Mylist([1,2,3])")
165
166 from cPickle import dumps
167
168 # We need to swap in our main module - this is only necessary
169 # inside the test framework, because IPython puts the interactive module
170 # in place (but the test framework undoes this).
171 _main = sys.modules['__main__']
172 sys.modules['__main__'] = ip.user_module
173 try:
174 res = dumps(ip.user_ns["w"])
175 finally:
176 sys.modules['__main__'] = _main
177 self.assertTrue(isinstance(res, bytes))
178
179 def test_global_ns(self):
180 "Code in functions must be able to access variables outside them."
181 ip = get_ipython()
182 ip.run_cell("a = 10")
183 ip.run_cell(("def f(x):"
184 " return x + a"))
185 ip.run_cell("b = f(12)")
186 self.assertEqual(ip.user_ns["b"], 22)
154 187
155 188 def test_bad_custom_tb(self):
156 189 """Check that InteractiveShell is protected from bad custom exception handlers"""
157 190 ip = get_ipython()
158 191 from IPython.utils import io
159 192 save_stderr = io.stderr
160 193 try:
161 194 # capture stderr
162 195 io.stderr = StringIO()
163 196 ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
164 197 self.assertEquals(ip.custom_exceptions, (IOError,))
165 198 ip.run_cell(u'raise IOError("foo")')
166 199 self.assertEquals(ip.custom_exceptions, ())
167 200 self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
168 201 finally:
169 202 io.stderr = save_stderr
170 203
171 204 def test_bad_custom_tb_return(self):
172 205 """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
173 206 ip = get_ipython()
174 207 from IPython.utils import io
175 208 save_stderr = io.stderr
176 209 try:
177 210 # capture stderr
178 211 io.stderr = StringIO()
179 212 ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
180 213 self.assertEquals(ip.custom_exceptions, (NameError,))
181 214 ip.run_cell(u'a=abracadabra')
182 215 self.assertEquals(ip.custom_exceptions, ())
183 216 self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
184 217 finally:
185 218 io.stderr = save_stderr
186 219
187 220 def test_drop_by_id(self):
188 221 ip = get_ipython()
189 222 myvars = {"a":object(), "b":object(), "c": object()}
190 223 ip.push(myvars, interactive=False)
191 224 for name in myvars:
192 225 assert name in ip.user_ns, name
193 226 assert name in ip.user_ns_hidden, name
194 227 ip.user_ns['b'] = 12
195 228 ip.drop_by_id(myvars)
196 229 for name in ["a", "c"]:
197 230 assert name not in ip.user_ns, name
198 231 assert name not in ip.user_ns_hidden, name
199 232 assert ip.user_ns['b'] == 12
200 233 ip.reset()
201 234
202 235 def test_var_expand(self):
203 236 ip = get_ipython()
204 237 ip.user_ns['f'] = u'Ca\xf1o'
205 238 self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
206 239
207 240 ip.user_ns['f'] = b'Ca\xc3\xb1o'
208 241 # This should not raise any exception:
209 242 ip.var_expand(u'echo $f')
210 243
211 244
212 245 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
213 246
214 247 def setUp(self):
215 248 self.BASETESTDIR = tempfile.mkdtemp()
216 249 self.TESTDIR = join(self.BASETESTDIR, u"Γ₯Àâ")
217 250 os.mkdir(self.TESTDIR)
218 251 with open(join(self.TESTDIR, u"Γ₯Àâtestscript.py"), "w") as sfile:
219 252 sfile.write("pass\n")
220 253 self.oldpath = os.getcwdu()
221 254 os.chdir(self.TESTDIR)
222 255 self.fname = u"Γ₯Àâtestscript.py"
223 256
224 257
225 258 def tearDown(self):
226 259 os.chdir(self.oldpath)
227 260 shutil.rmtree(self.BASETESTDIR)
228 261
229 262 def test_1(self):
230 263 """Test safe_execfile with non-ascii path
231 264 """
232 265 _ip.shell.safe_execfile(self.fname, {}, raise_exceptions=True)
233 266
234 267
235 268 class TestSystemRaw(unittest.TestCase):
236 269 def test_1(self):
237 270 """Test system_raw with non-ascii cmd
238 271 """
239 272 cmd = ur'''python -c "'Γ₯Àâ'" '''
240 273 _ip.shell.system_raw(cmd)
@@ -1,239 +1,238 b''
1 1 """Global IPython app to support test running.
2 2
3 3 We must start our own ipython object and heavily muck with it so that all the
4 4 modifications IPython makes to system behavior don't send the doctest machinery
5 5 into a fit. This code should be considered a gross hack, but it gets the job
6 6 done.
7 7 """
8 8 from __future__ import absolute_import
9 9 from __future__ import print_function
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Copyright (C) 2009-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21
22 22 # stdlib
23 23 import __builtin__ as builtin_mod
24 24 import os
25 25 import sys
26 26
27 27 # our own
28 28 from . import tools
29 29
30 30 from IPython.core import page
31 31 from IPython.utils import io
32 32 from IPython.utils import py3compat
33 33 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Functions
37 37 #-----------------------------------------------------------------------------
38 38
39 39 class StreamProxy(io.IOStream):
40 40 """Proxy for sys.stdout/err. This will request the stream *at call time*
41 41 allowing for nose's Capture plugin's redirection of sys.stdout/err.
42 42
43 43 Parameters
44 44 ----------
45 45 name : str
46 46 The name of the stream. This will be requested anew at every call
47 47 """
48 48
49 49 def __init__(self, name):
50 50 self.name=name
51 51
52 52 @property
53 53 def stream(self):
54 54 return getattr(sys, self.name)
55 55
56 56 def flush(self):
57 57 self.stream.flush()
58 58
59 59 # Hack to modify the %run command so we can sync the user's namespace with the
60 60 # test globals. Once we move over to a clean magic system, this will be done
61 61 # with much less ugliness.
62 62
63 63 class py_file_finder(object):
64 64 def __init__(self,test_filename):
65 65 self.test_filename = test_filename
66 66
67 67 def __call__(self,name,win32=False):
68 68 from IPython.utils.path import get_py_filename
69 69 try:
70 70 return get_py_filename(name,win32=win32)
71 71 except IOError:
72 72 test_dir = os.path.dirname(self.test_filename)
73 73 new_path = os.path.join(test_dir,name)
74 74 return get_py_filename(new_path,win32=win32)
75 75
76 76
77 77 def _run_ns_sync(self,arg_s,runner=None):
78 78 """Modified version of %run that syncs testing namespaces.
79 79
80 80 This is strictly needed for running doctests that call %run.
81 81 """
82 82 #print('in run_ns_sync', arg_s, file=sys.stderr) # dbg
83 83 finder = py_file_finder(arg_s)
84 84 return get_ipython().magic_run_ori(arg_s, runner, finder)
85 85
86 86
87 87 class ipnsdict(dict):
88 88 """A special subclass of dict for use as an IPython namespace in doctests.
89 89
90 90 This subclass adds a simple checkpointing capability so that when testing
91 91 machinery clears it (we use it as the test execution context), it doesn't
92 92 get completely destroyed.
93 93
94 94 In addition, it can handle the presence of the '_' key in a special manner,
95 95 which is needed because of how Python's doctest machinery operates with
96 96 '_'. See constructor and :meth:`update` for details.
97 97 """
98 98
99 99 def __init__(self,*a):
100 100 dict.__init__(self,*a)
101 101 self._savedict = {}
102 102 # If this flag is True, the .update() method will unconditionally
103 103 # remove a key named '_'. This is so that such a dict can be used as a
104 104 # namespace in doctests that call '_'.
105 105 self.protect_underscore = False
106 106
107 107 def clear(self):
108 108 dict.clear(self)
109 109 self.update(self._savedict)
110 110
111 111 def _checkpoint(self):
112 112 self._savedict.clear()
113 113 self._savedict.update(self)
114 114
115 115 def update(self,other):
116 116 self._checkpoint()
117 117 dict.update(self,other)
118 118
119 119 if self.protect_underscore:
120 120 # If '_' is in the namespace, python won't set it when executing
121 121 # code *in doctests*, and we have multiple doctests that use '_'.
122 122 # So we ensure that the namespace is always 'clean' of it before
123 123 # it's used for test code execution.
124 124 # This flag is only turned on by the doctest machinery, so that
125 125 # normal test code can assume the _ key is updated like any other
126 126 # key and can test for its presence after cell executions.
127 127 self.pop('_', None)
128 128
129 129 # The builtins namespace must *always* be the real __builtin__ module,
130 130 # else weird stuff happens. The main ipython code does have provisions
131 131 # to ensure this after %run, but since in this class we do some
132 132 # aggressive low-level cleaning of the execution namespace, we need to
133 133 # correct for that ourselves, to ensure consitency with the 'real'
134 134 # ipython.
135 135 self['__builtins__'] = builtin_mod
136 136
137 137 def __delitem__(self, key):
138 138 """Part of the test suite checks that we can release all
139 139 references to an object. So we need to make sure that we're not
140 140 keeping a reference in _savedict."""
141 141 dict.__delitem__(self, key)
142 142 try:
143 143 del self._savedict[key]
144 144 except KeyError:
145 145 pass
146 146
147 147
148 148 def get_ipython():
149 149 # This will get replaced by the real thing once we start IPython below
150 150 return start_ipython()
151 151
152 152
153 153 # A couple of methods to override those in the running IPython to interact
154 154 # better with doctest (doctest captures on raw stdout, so we need to direct
155 155 # various types of output there otherwise it will miss them).
156 156
157 157 def xsys(self, cmd):
158 158 """Replace the default system call with a capturing one for doctest.
159 159 """
160 160 # We use getoutput, but we need to strip it because pexpect captures
161 161 # the trailing newline differently from commands.getoutput
162 162 print(self.getoutput(cmd, split=False).rstrip(), end='', file=sys.stdout)
163 163 sys.stdout.flush()
164 164
165 165
166 166 def _showtraceback(self, etype, evalue, stb):
167 167 """Print the traceback purely on stdout for doctest to capture it.
168 168 """
169 169 print(self.InteractiveTB.stb2text(stb), file=sys.stdout)
170 170
171 171
172 172 def start_ipython():
173 173 """Start a global IPython shell, which we need for IPython-specific syntax.
174 174 """
175 175 global get_ipython
176 176
177 177 # This function should only ever run once!
178 178 if hasattr(start_ipython, 'already_called'):
179 179 return
180 180 start_ipython.already_called = True
181 181
182 182 # Store certain global objects that IPython modifies
183 183 _displayhook = sys.displayhook
184 184 _excepthook = sys.excepthook
185 185 _main = sys.modules.get('__main__')
186 186
187 187 # Create custom argv and namespaces for our IPython to be test-friendly
188 188 config = tools.default_config()
189 189
190 190 # Create and initialize our test-friendly IPython instance.
191 191 shell = TerminalInteractiveShell.instance(config=config,
192 192 user_ns=ipnsdict(),
193 193 )
194 194
195 195 # A few more tweaks needed for playing nicely with doctests...
196 196
197 197 # remove history file
198 198 shell.tempfiles.append(config.HistoryManager.hist_file)
199 199
200 200 # These traps are normally only active for interactive use, set them
201 201 # permanently since we'll be mocking interactive sessions.
202 202 shell.builtin_trap.activate()
203 203
204 204 # Modify the IPython system call with one that uses getoutput, so that we
205 205 # can capture subcommands and print them to Python's stdout, otherwise the
206 206 # doctest machinery would miss them.
207 207 shell.system = py3compat.MethodType(xsys, shell)
208
209
208
210 209 shell._showtraceback = py3compat.MethodType(_showtraceback, shell)
211 210
212 211 # IPython is ready, now clean up some global state...
213 212
214 213 # Deactivate the various python system hooks added by ipython for
215 214 # interactive convenience so we don't confuse the doctest system
216 215 sys.modules['__main__'] = _main
217 216 sys.displayhook = _displayhook
218 217 sys.excepthook = _excepthook
219 218
220 219 # So that ipython magics and aliases can be doctested (they work by making
221 220 # a call into a global _ip object). Also make the top-level get_ipython
222 221 # now return this without recursively calling here again.
223 222 _ip = shell
224 223 get_ipython = _ip.get_ipython
225 224 builtin_mod._ip = _ip
226 225 builtin_mod.get_ipython = get_ipython
227 226
228 227 # To avoid extra IPython messages during testing, suppress io.stdout/stderr
229 228 io.stdout = StreamProxy('stdout')
230 229 io.stderr = StreamProxy('stderr')
231 230
232 231 # Override paging, so we don't require user interaction during the tests.
233 232 def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
234 233 print(strng)
235 234
236 235 page.orig_page = page.page
237 236 page.page = nopage
238 237
239 238 return _ip
@@ -1,810 +1,814 b''
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by setting the
7 7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 8 by interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import __builtin__
23 23 import commands
24 24 import doctest
25 25 import inspect
26 26 import logging
27 27 import os
28 28 import re
29 29 import sys
30 30 import traceback
31 31 import unittest
32 32
33 33 from inspect import getmodule
34 34 from StringIO import StringIO
35 35
36 36 # We are overriding the default doctest runner, so we need to import a few
37 37 # things from doctest directly
38 38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 39 _unittest_reportflags, DocTestRunner,
40 40 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 41 _exception_traceback,
42 42 linecache)
43 43
44 44 # Third-party modules
45 45 import nose.core
46 46
47 47 from nose.plugins import doctests, Plugin
48 48 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module globals and other constants
52 52 #-----------------------------------------------------------------------------
53 53
54 54 log = logging.getLogger(__name__)
55 55
56 56
57 57 #-----------------------------------------------------------------------------
58 58 # Classes and functions
59 59 #-----------------------------------------------------------------------------
60 60
61 61 def is_extension_module(filename):
62 62 """Return whether the given filename is an extension module.
63 63
64 64 This simply checks that the extension is either .so or .pyd.
65 65 """
66 66 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
67 67
68 68
69 69 class DocTestSkip(object):
70 70 """Object wrapper for doctests to be skipped."""
71 71
72 72 ds_skip = """Doctest to skip.
73 73 >>> 1 #doctest: +SKIP
74 74 """
75 75
76 76 def __init__(self,obj):
77 77 self.obj = obj
78 78
79 79 def __getattribute__(self,key):
80 80 if key == '__doc__':
81 81 return DocTestSkip.ds_skip
82 82 else:
83 83 return getattr(object.__getattribute__(self,'obj'),key)
84 84
85 85 # Modified version of the one in the stdlib, that fixes a python bug (doctests
86 86 # not found in extension modules, http://bugs.python.org/issue3158)
87 87 class DocTestFinder(doctest.DocTestFinder):
88 88
89 89 def _from_module(self, module, object):
90 90 """
91 91 Return true if the given object is defined in the given
92 92 module.
93 93 """
94 94 if module is None:
95 95 return True
96 96 elif inspect.isfunction(object):
97 97 return module.__dict__ is object.func_globals
98 98 elif inspect.isbuiltin(object):
99 99 return module.__name__ == object.__module__
100 100 elif inspect.isclass(object):
101 101 return module.__name__ == object.__module__
102 102 elif inspect.ismethod(object):
103 103 # This one may be a bug in cython that fails to correctly set the
104 104 # __module__ attribute of methods, but since the same error is easy
105 105 # to make by extension code writers, having this safety in place
106 106 # isn't such a bad idea
107 107 return module.__name__ == object.im_class.__module__
108 108 elif inspect.getmodule(object) is not None:
109 109 return module is inspect.getmodule(object)
110 110 elif hasattr(object, '__module__'):
111 111 return module.__name__ == object.__module__
112 112 elif isinstance(object, property):
113 113 return True # [XX] no way not be sure.
114 114 else:
115 115 raise ValueError("object must be a class or function")
116 116
117 117 def _find(self, tests, obj, name, module, source_lines, globs, seen):
118 118 """
119 119 Find tests for the given object and any contained objects, and
120 120 add them to `tests`.
121 121 """
122 122 #print '_find for:', obj, name, module # dbg
123 123 if hasattr(obj,"skip_doctest"):
124 124 #print 'SKIPPING DOCTEST FOR:',obj # dbg
125 125 obj = DocTestSkip(obj)
126 126
127 127 doctest.DocTestFinder._find(self,tests, obj, name, module,
128 128 source_lines, globs, seen)
129 129
130 130 # Below we re-run pieces of the above method with manual modifications,
131 131 # because the original code is buggy and fails to correctly identify
132 132 # doctests in extension modules.
133 133
134 134 # Local shorthands
135 135 from inspect import isroutine, isclass, ismodule
136 136
137 137 # Look for tests in a module's contained objects.
138 138 if inspect.ismodule(obj) and self._recurse:
139 139 for valname, val in obj.__dict__.items():
140 140 valname1 = '%s.%s' % (name, valname)
141 141 if ( (isroutine(val) or isclass(val))
142 142 and self._from_module(module, val) ):
143 143
144 144 self._find(tests, val, valname1, module, source_lines,
145 145 globs, seen)
146 146
147 147 # Look for tests in a class's contained objects.
148 148 if inspect.isclass(obj) and self._recurse:
149 149 #print 'RECURSE into class:',obj # dbg
150 150 for valname, val in obj.__dict__.items():
151 151 # Special handling for staticmethod/classmethod.
152 152 if isinstance(val, staticmethod):
153 153 val = getattr(obj, valname)
154 154 if isinstance(val, classmethod):
155 155 val = getattr(obj, valname).im_func
156 156
157 157 # Recurse to methods, properties, and nested classes.
158 158 if ((inspect.isfunction(val) or inspect.isclass(val) or
159 159 inspect.ismethod(val) or
160 160 isinstance(val, property)) and
161 161 self._from_module(module, val)):
162 162 valname = '%s.%s' % (name, valname)
163 163 self._find(tests, val, valname, module, source_lines,
164 164 globs, seen)
165 165
166 166
167 167 class IPDoctestOutputChecker(doctest.OutputChecker):
168 168 """Second-chance checker with support for random tests.
169 169
170 170 If the default comparison doesn't pass, this checker looks in the expected
171 171 output string for flags that tell us to ignore the output.
172 172 """
173 173
174 174 random_re = re.compile(r'#\s*random\s+')
175 175
176 176 def check_output(self, want, got, optionflags):
177 177 """Check output, accepting special markers embedded in the output.
178 178
179 179 If the output didn't pass the default validation but the special string
180 180 '#random' is included, we accept it."""
181 181
182 182 # Let the original tester verify first, in case people have valid tests
183 183 # that happen to have a comment saying '#random' embedded in.
184 184 ret = doctest.OutputChecker.check_output(self, want, got,
185 185 optionflags)
186 186 if not ret and self.random_re.search(want):
187 187 #print >> sys.stderr, 'RANDOM OK:',want # dbg
188 188 return True
189 189
190 190 return ret
191 191
192 192
193 193 class DocTestCase(doctests.DocTestCase):
194 194 """Proxy for DocTestCase: provides an address() method that
195 195 returns the correct address for the doctest case. Otherwise
196 196 acts as a proxy to the test case. To provide hints for address(),
197 197 an obj may also be passed -- this will be used as the test object
198 198 for purposes of determining the test address, if it is provided.
199 199 """
200 200
201 201 # Note: this method was taken from numpy's nosetester module.
202 202
203 203 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
204 204 # its constructor that blocks non-default arguments from being passed
205 205 # down into doctest.DocTestCase
206 206
207 207 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
208 208 checker=None, obj=None, result_var='_'):
209 209 self._result_var = result_var
210 210 doctests.DocTestCase.__init__(self, test,
211 211 optionflags=optionflags,
212 212 setUp=setUp, tearDown=tearDown,
213 213 checker=checker)
214 214 # Now we must actually copy the original constructor from the stdlib
215 215 # doctest class, because we can't call it directly and a bug in nose
216 216 # means it never gets passed the right arguments.
217 217
218 218 self._dt_optionflags = optionflags
219 219 self._dt_checker = checker
220 220 self._dt_test = test
221 221 self._dt_test_globs_ori = test.globs
222 222 self._dt_setUp = setUp
223 223 self._dt_tearDown = tearDown
224 224
225 225 # XXX - store this runner once in the object!
226 226 runner = IPDocTestRunner(optionflags=optionflags,
227 227 checker=checker, verbose=False)
228 228 self._dt_runner = runner
229 229
230 230
231 231 # Each doctest should remember the directory it was loaded from, so
232 232 # things like %run work without too many contortions
233 233 self._ori_dir = os.path.dirname(test.filename)
234 234
235 235 # Modified runTest from the default stdlib
236 236 def runTest(self):
237 237 test = self._dt_test
238 238 runner = self._dt_runner
239 239
240 240 old = sys.stdout
241 241 new = StringIO()
242 242 optionflags = self._dt_optionflags
243 243
244 244 if not (optionflags & REPORTING_FLAGS):
245 245 # The option flags don't include any reporting flags,
246 246 # so add the default reporting flags
247 247 optionflags |= _unittest_reportflags
248 248
249 249 try:
250 250 # Save our current directory and switch out to the one where the
251 251 # test was originally created, in case another doctest did a
252 252 # directory change. We'll restore this in the finally clause.
253 253 curdir = os.getcwdu()
254 254 #print 'runTest in dir:', self._ori_dir # dbg
255 255 os.chdir(self._ori_dir)
256 256
257 257 runner.DIVIDER = "-"*70
258 258 failures, tries = runner.run(test,out=new.write,
259 259 clear_globs=False)
260 260 finally:
261 261 sys.stdout = old
262 262 os.chdir(curdir)
263 263
264 264 if failures:
265 265 raise self.failureException(self.format_failure(new.getvalue()))
266 266
267 267 def setUp(self):
268 268 """Modified test setup that syncs with ipython namespace"""
269 269 #print "setUp test", self._dt_test.examples # dbg
270 270 if isinstance(self._dt_test.examples[0],IPExample):
271 271 # for IPython examples *only*, we swap the globals with the ipython
272 272 # namespace, after updating it with the globals (which doctest
273 273 # fills with the necessary info from the module being tested).
274 self.user_ns_orig = {}
275 self.user_ns_orig.update(_ip.user_ns)
274 276 _ip.user_ns.update(self._dt_test.globs)
275 277 self._dt_test.globs = _ip.user_ns
276 278 # IPython must protect the _ key in the namespace (it can't exist)
277 279 # so that Python's doctest code sets it naturally, so we enable
278 280 # this feature of our testing namespace.
279 281 _ip.user_ns.protect_underscore = True
280 282
281 283 super(DocTestCase, self).setUp()
282 284
283 285 def tearDown(self):
284 286
285 287 # Undo the test.globs reassignment we made, so that the parent class
286 288 # teardown doesn't destroy the ipython namespace
287 289 if isinstance(self._dt_test.examples[0],IPExample):
288 290 self._dt_test.globs = self._dt_test_globs_ori
291 _ip.user_ns.clear()
292 _ip.user_ns.update(self.user_ns_orig)
289 293 # Restore the behavior of the '_' key in the user namespace to
290 294 # normal after each doctest, so that unittests behave normally
291 295 _ip.user_ns.protect_underscore = False
292 296
293 297 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
294 298 # it does look like one to me: its tearDown method tries to run
295 299 #
296 300 # delattr(__builtin__, self._result_var)
297 301 #
298 302 # without checking that the attribute really is there; it implicitly
299 303 # assumes it should have been set via displayhook. But if the
300 304 # displayhook was never called, this doesn't necessarily happen. I
301 305 # haven't been able to find a little self-contained example outside of
302 306 # ipython that would show the problem so I can report it to the nose
303 307 # team, but it does happen a lot in our code.
304 308 #
305 309 # So here, we just protect as narrowly as possible by trapping an
306 310 # attribute error whose message would be the name of self._result_var,
307 311 # and letting any other error propagate.
308 312 try:
309 313 super(DocTestCase, self).tearDown()
310 314 except AttributeError, exc:
311 315 if exc.args[0] != self._result_var:
312 316 raise
313 317
314 318
315 319 # A simple subclassing of the original with a different class name, so we can
316 320 # distinguish and treat differently IPython examples from pure python ones.
317 321 class IPExample(doctest.Example): pass
318 322
319 323
320 324 class IPExternalExample(doctest.Example):
321 325 """Doctest examples to be run in an external process."""
322 326
323 327 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
324 328 options=None):
325 329 # Parent constructor
326 330 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
327 331
328 332 # An EXTRA newline is needed to prevent pexpect hangs
329 333 self.source += '\n'
330 334
331 335
332 336 class IPDocTestParser(doctest.DocTestParser):
333 337 """
334 338 A class used to parse strings containing doctest examples.
335 339
336 340 Note: This is a version modified to properly recognize IPython input and
337 341 convert any IPython examples into valid Python ones.
338 342 """
339 343 # This regular expression is used to find doctest examples in a
340 344 # string. It defines three groups: `source` is the source code
341 345 # (including leading indentation and prompts); `indent` is the
342 346 # indentation of the first (PS1) line of the source code; and
343 347 # `want` is the expected output (including leading indentation).
344 348
345 349 # Classic Python prompts or default IPython ones
346 350 _PS1_PY = r'>>>'
347 351 _PS2_PY = r'\.\.\.'
348 352
349 353 _PS1_IP = r'In\ \[\d+\]:'
350 354 _PS2_IP = r'\ \ \ \.\.\.+:'
351 355
352 356 _RE_TPL = r'''
353 357 # Source consists of a PS1 line followed by zero or more PS2 lines.
354 358 (?P<source>
355 359 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
356 360 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
357 361 \n? # a newline
358 362 # Want consists of any non-blank lines that do not start with PS1.
359 363 (?P<want> (?:(?![ ]*$) # Not a blank line
360 364 (?![ ]*%s) # Not a line starting with PS1
361 365 (?![ ]*%s) # Not a line starting with PS2
362 366 .*$\n? # But any other line
363 367 )*)
364 368 '''
365 369
366 370 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
367 371 re.MULTILINE | re.VERBOSE)
368 372
369 373 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
370 374 re.MULTILINE | re.VERBOSE)
371 375
372 376 # Mark a test as being fully random. In this case, we simply append the
373 377 # random marker ('#random') to each individual example's output. This way
374 378 # we don't need to modify any other code.
375 379 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
376 380
377 381 # Mark tests to be executed in an external process - currently unsupported.
378 382 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
379 383
380 384 def ip2py(self,source):
381 385 """Convert input IPython source into valid Python."""
382 386 out = []
383 387 newline = out.append
384 388 #print 'IPSRC:\n',source,'\n###' # dbg
385 389 # The input source must be first stripped of all bracketing whitespace
386 390 # and turned into lines, so it looks to the parser like regular user
387 391 # input
388 392 for lnum,line in enumerate(source.strip().splitlines()):
389 393 newline(_ip.prefilter(line,lnum>0))
390 394 newline('') # ensure a closing newline, needed by doctest
391 395 #print "PYSRC:", '\n'.join(out) # dbg
392 396 return '\n'.join(out)
393 397
394 398 def parse(self, string, name='<string>'):
395 399 """
396 400 Divide the given string into examples and intervening text,
397 401 and return them as a list of alternating Examples and strings.
398 402 Line numbers for the Examples are 0-based. The optional
399 403 argument `name` is a name identifying this string, and is only
400 404 used for error messages.
401 405 """
402 406
403 407 #print 'Parse string:\n',string # dbg
404 408
405 409 string = string.expandtabs()
406 410 # If all lines begin with the same indentation, then strip it.
407 411 min_indent = self._min_indent(string)
408 412 if min_indent > 0:
409 413 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
410 414
411 415 output = []
412 416 charno, lineno = 0, 0
413 417
414 418 # We make 'all random' tests by adding the '# random' mark to every
415 419 # block of output in the test.
416 420 if self._RANDOM_TEST.search(string):
417 421 random_marker = '\n# random'
418 422 else:
419 423 random_marker = ''
420 424
421 425 # Whether to convert the input from ipython to python syntax
422 426 ip2py = False
423 427 # Find all doctest examples in the string. First, try them as Python
424 428 # examples, then as IPython ones
425 429 terms = list(self._EXAMPLE_RE_PY.finditer(string))
426 430 if terms:
427 431 # Normal Python example
428 432 #print '-'*70 # dbg
429 433 #print 'PyExample, Source:\n',string # dbg
430 434 #print '-'*70 # dbg
431 435 Example = doctest.Example
432 436 else:
433 437 # It's an ipython example. Note that IPExamples are run
434 438 # in-process, so their syntax must be turned into valid python.
435 439 # IPExternalExamples are run out-of-process (via pexpect) so they
436 440 # don't need any filtering (a real ipython will be executing them).
437 441 terms = list(self._EXAMPLE_RE_IP.finditer(string))
438 442 if self._EXTERNAL_IP.search(string):
439 443 #print '-'*70 # dbg
440 444 #print 'IPExternalExample, Source:\n',string # dbg
441 445 #print '-'*70 # dbg
442 446 Example = IPExternalExample
443 447 else:
444 448 #print '-'*70 # dbg
445 449 #print 'IPExample, Source:\n',string # dbg
446 450 #print '-'*70 # dbg
447 451 Example = IPExample
448 452 ip2py = True
449 453
450 454 for m in terms:
451 455 # Add the pre-example text to `output`.
452 456 output.append(string[charno:m.start()])
453 457 # Update lineno (lines before this example)
454 458 lineno += string.count('\n', charno, m.start())
455 459 # Extract info from the regexp match.
456 460 (source, options, want, exc_msg) = \
457 461 self._parse_example(m, name, lineno,ip2py)
458 462
459 463 # Append the random-output marker (it defaults to empty in most
460 464 # cases, it's only non-empty for 'all-random' tests):
461 465 want += random_marker
462 466
463 467 if Example is IPExternalExample:
464 468 options[doctest.NORMALIZE_WHITESPACE] = True
465 469 want += '\n'
466 470
467 471 # Create an Example, and add it to the list.
468 472 if not self._IS_BLANK_OR_COMMENT(source):
469 473 output.append(Example(source, want, exc_msg,
470 474 lineno=lineno,
471 475 indent=min_indent+len(m.group('indent')),
472 476 options=options))
473 477 # Update lineno (lines inside this example)
474 478 lineno += string.count('\n', m.start(), m.end())
475 479 # Update charno.
476 480 charno = m.end()
477 481 # Add any remaining post-example text to `output`.
478 482 output.append(string[charno:])
479 483 return output
480 484
481 485 def _parse_example(self, m, name, lineno,ip2py=False):
482 486 """
483 487 Given a regular expression match from `_EXAMPLE_RE` (`m`),
484 488 return a pair `(source, want)`, where `source` is the matched
485 489 example's source code (with prompts and indentation stripped);
486 490 and `want` is the example's expected output (with indentation
487 491 stripped).
488 492
489 493 `name` is the string's name, and `lineno` is the line number
490 494 where the example starts; both are used for error messages.
491 495
492 496 Optional:
493 497 `ip2py`: if true, filter the input via IPython to convert the syntax
494 498 into valid python.
495 499 """
496 500
497 501 # Get the example's indentation level.
498 502 indent = len(m.group('indent'))
499 503
500 504 # Divide source into lines; check that they're properly
501 505 # indented; and then strip their indentation & prompts.
502 506 source_lines = m.group('source').split('\n')
503 507
504 508 # We're using variable-length input prompts
505 509 ps1 = m.group('ps1')
506 510 ps2 = m.group('ps2')
507 511 ps1_len = len(ps1)
508 512
509 513 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
510 514 if ps2:
511 515 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
512 516
513 517 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
514 518
515 519 if ip2py:
516 520 # Convert source input from IPython into valid Python syntax
517 521 source = self.ip2py(source)
518 522
519 523 # Divide want into lines; check that it's properly indented; and
520 524 # then strip the indentation. Spaces before the last newline should
521 525 # be preserved, so plain rstrip() isn't good enough.
522 526 want = m.group('want')
523 527 want_lines = want.split('\n')
524 528 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
525 529 del want_lines[-1] # forget final newline & spaces after it
526 530 self._check_prefix(want_lines, ' '*indent, name,
527 531 lineno + len(source_lines))
528 532
529 533 # Remove ipython output prompt that might be present in the first line
530 534 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
531 535
532 536 want = '\n'.join([wl[indent:] for wl in want_lines])
533 537
534 538 # If `want` contains a traceback message, then extract it.
535 539 m = self._EXCEPTION_RE.match(want)
536 540 if m:
537 541 exc_msg = m.group('msg')
538 542 else:
539 543 exc_msg = None
540 544
541 545 # Extract options from the source.
542 546 options = self._find_options(source, name, lineno)
543 547
544 548 return source, options, want, exc_msg
545 549
546 550 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
547 551 """
548 552 Given the lines of a source string (including prompts and
549 553 leading indentation), check to make sure that every prompt is
550 554 followed by a space character. If any line is not followed by
551 555 a space character, then raise ValueError.
552 556
553 557 Note: IPython-modified version which takes the input prompt length as a
554 558 parameter, so that prompts of variable length can be dealt with.
555 559 """
556 560 space_idx = indent+ps1_len
557 561 min_len = space_idx+1
558 562 for i, line in enumerate(lines):
559 563 if len(line) >= min_len and line[space_idx] != ' ':
560 564 raise ValueError('line %r of the docstring for %s '
561 565 'lacks blank after %s: %r' %
562 566 (lineno+i+1, name,
563 567 line[indent:space_idx], line))
564 568
565 569
566 570 SKIP = doctest.register_optionflag('SKIP')
567 571
568 572
569 573 class IPDocTestRunner(doctest.DocTestRunner,object):
570 574 """Test runner that synchronizes the IPython namespace with test globals.
571 575 """
572 576
573 577 def run(self, test, compileflags=None, out=None, clear_globs=True):
574 578
575 579 # Hack: ipython needs access to the execution context of the example,
576 580 # so that it can propagate user variables loaded by %run into
577 581 # test.globs. We put them here into our modified %run as a function
578 582 # attribute. Our new %run will then only make the namespace update
579 583 # when called (rather than unconconditionally updating test.globs here
580 584 # for all examples, most of which won't be calling %run anyway).
581 585 #_ip._ipdoctest_test_globs = test.globs
582 586 #_ip._ipdoctest_test_filename = test.filename
583 587
584 588 test.globs.update(_ip.user_ns)
585 589
586 590 return super(IPDocTestRunner,self).run(test,
587 591 compileflags,out,clear_globs)
588 592
589 593
590 594 class DocFileCase(doctest.DocFileCase):
591 595 """Overrides to provide filename
592 596 """
593 597 def address(self):
594 598 return (self._dt_test.filename, None, None)
595 599
596 600
597 601 class ExtensionDoctest(doctests.Doctest):
598 602 """Nose Plugin that supports doctests in extension modules.
599 603 """
600 604 name = 'extdoctest' # call nosetests with --with-extdoctest
601 605 enabled = True
602 606
603 607 def __init__(self,exclude_patterns=None):
604 608 """Create a new ExtensionDoctest plugin.
605 609
606 610 Parameters
607 611 ----------
608 612
609 613 exclude_patterns : sequence of strings, optional
610 614 These patterns are compiled as regular expressions, subsequently used
611 615 to exclude any filename which matches them from inclusion in the test
612 616 suite (using pattern.search(), NOT pattern.match() ).
613 617 """
614 618
615 619 if exclude_patterns is None:
616 620 exclude_patterns = []
617 621 self.exclude_patterns = map(re.compile,exclude_patterns)
618 622 doctests.Doctest.__init__(self)
619 623
620 624 def options(self, parser, env=os.environ):
621 625 Plugin.options(self, parser, env)
622 626 parser.add_option('--doctest-tests', action='store_true',
623 627 dest='doctest_tests',
624 628 default=env.get('NOSE_DOCTEST_TESTS',True),
625 629 help="Also look for doctests in test modules. "
626 630 "Note that classes, methods and functions should "
627 631 "have either doctests or non-doctest tests, "
628 632 "not both. [NOSE_DOCTEST_TESTS]")
629 633 parser.add_option('--doctest-extension', action="append",
630 634 dest="doctestExtension",
631 635 help="Also look for doctests in files with "
632 636 "this extension [NOSE_DOCTEST_EXTENSION]")
633 637 # Set the default as a list, if given in env; otherwise
634 638 # an additional value set on the command line will cause
635 639 # an error.
636 640 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
637 641 if env_setting is not None:
638 642 parser.set_defaults(doctestExtension=tolist(env_setting))
639 643
640 644
641 645 def configure(self, options, config):
642 646 Plugin.configure(self, options, config)
643 647 # Pull standard doctest plugin out of config; we will do doctesting
644 648 config.plugins.plugins = [p for p in config.plugins.plugins
645 649 if p.name != 'doctest']
646 650 self.doctest_tests = options.doctest_tests
647 651 self.extension = tolist(options.doctestExtension)
648 652
649 653 self.parser = doctest.DocTestParser()
650 654 self.finder = DocTestFinder()
651 655 self.checker = IPDoctestOutputChecker()
652 656 self.globs = None
653 657 self.extraglobs = None
654 658
655 659
656 660 def loadTestsFromExtensionModule(self,filename):
657 661 bpath,mod = os.path.split(filename)
658 662 modname = os.path.splitext(mod)[0]
659 663 try:
660 664 sys.path.append(bpath)
661 665 module = __import__(modname)
662 666 tests = list(self.loadTestsFromModule(module))
663 667 finally:
664 668 sys.path.pop()
665 669 return tests
666 670
667 671 # NOTE: the method below is almost a copy of the original one in nose, with
668 672 # a few modifications to control output checking.
669 673
670 674 def loadTestsFromModule(self, module):
671 675 #print '*** ipdoctest - lTM',module # dbg
672 676
673 677 if not self.matches(module.__name__):
674 678 log.debug("Doctest doesn't want module %s", module)
675 679 return
676 680
677 681 tests = self.finder.find(module,globs=self.globs,
678 682 extraglobs=self.extraglobs)
679 683 if not tests:
680 684 return
681 685
682 686 # always use whitespace and ellipsis options
683 687 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
684 688
685 689 tests.sort()
686 690 module_file = module.__file__
687 691 if module_file[-4:] in ('.pyc', '.pyo'):
688 692 module_file = module_file[:-1]
689 693 for test in tests:
690 694 if not test.examples:
691 695 continue
692 696 if not test.filename:
693 697 test.filename = module_file
694 698
695 699 yield DocTestCase(test,
696 700 optionflags=optionflags,
697 701 checker=self.checker)
698 702
699 703
700 704 def loadTestsFromFile(self, filename):
701 705 #print "ipdoctest - from file", filename # dbg
702 706 if is_extension_module(filename):
703 707 for t in self.loadTestsFromExtensionModule(filename):
704 708 yield t
705 709 else:
706 710 if self.extension and anyp(filename.endswith, self.extension):
707 711 name = os.path.basename(filename)
708 712 dh = open(filename)
709 713 try:
710 714 doc = dh.read()
711 715 finally:
712 716 dh.close()
713 717 test = self.parser.get_doctest(
714 718 doc, globs={'__file__': filename}, name=name,
715 719 filename=filename, lineno=0)
716 720 if test.examples:
717 721 #print 'FileCase:',test.examples # dbg
718 722 yield DocFileCase(test)
719 723 else:
720 724 yield False # no tests to load
721 725
722 726 def wantFile(self,filename):
723 727 """Return whether the given filename should be scanned for tests.
724 728
725 729 Modified version that accepts extension modules as valid containers for
726 730 doctests.
727 731 """
728 732 #print '*** ipdoctest- wantFile:',filename # dbg
729 733
730 734 for pat in self.exclude_patterns:
731 735 if pat.search(filename):
732 736 # print '###>>> SKIP:',filename # dbg
733 737 return False
734 738
735 739 if is_extension_module(filename):
736 740 return True
737 741 else:
738 742 return doctests.Doctest.wantFile(self,filename)
739 743
740 744 def wantDirectory(self, directory):
741 745 """Return whether the given directory should be scanned for tests.
742 746
743 747 Modified version that supports exclusions.
744 748 """
745 749
746 750 for pat in self.exclude_patterns:
747 751 if pat.search(directory):
748 752 return False
749 753 return True
750 754
751 755
752 756 class IPythonDoctest(ExtensionDoctest):
753 757 """Nose Plugin that supports doctests in extension modules.
754 758 """
755 759 name = 'ipdoctest' # call nosetests with --with-ipdoctest
756 760 enabled = True
757 761
758 762 def makeTest(self, obj, parent):
759 763 """Look for doctests in the given object, which will be a
760 764 function, method or class.
761 765 """
762 766 #print 'Plugin analyzing:', obj, parent # dbg
763 767 # always use whitespace and ellipsis options
764 768 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
765 769
766 770 doctests = self.finder.find(obj, module=getmodule(parent))
767 771 if doctests:
768 772 for test in doctests:
769 773 if len(test.examples) == 0:
770 774 continue
771 775
772 776 yield DocTestCase(test, obj=obj,
773 777 optionflags=optionflags,
774 778 checker=self.checker)
775 779
776 780 def options(self, parser, env=os.environ):
777 781 #print "Options for nose plugin:", self.name # dbg
778 782 Plugin.options(self, parser, env)
779 783 parser.add_option('--ipdoctest-tests', action='store_true',
780 784 dest='ipdoctest_tests',
781 785 default=env.get('NOSE_IPDOCTEST_TESTS',True),
782 786 help="Also look for doctests in test modules. "
783 787 "Note that classes, methods and functions should "
784 788 "have either doctests or non-doctest tests, "
785 789 "not both. [NOSE_IPDOCTEST_TESTS]")
786 790 parser.add_option('--ipdoctest-extension', action="append",
787 791 dest="ipdoctest_extension",
788 792 help="Also look for doctests in files with "
789 793 "this extension [NOSE_IPDOCTEST_EXTENSION]")
790 794 # Set the default as a list, if given in env; otherwise
791 795 # an additional value set on the command line will cause
792 796 # an error.
793 797 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
794 798 if env_setting is not None:
795 799 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
796 800
797 801 def configure(self, options, config):
798 802 #print "Configuring nose plugin:", self.name # dbg
799 803 Plugin.configure(self, options, config)
800 804 # Pull standard doctest plugin out of config; we will do doctesting
801 805 config.plugins.plugins = [p for p in config.plugins.plugins
802 806 if p.name != 'doctest']
803 807 self.doctest_tests = options.ipdoctest_tests
804 808 self.extension = tolist(options.ipdoctest_extension)
805 809
806 810 self.parser = IPDocTestParser()
807 811 self.finder = DocTestFinder(parser=self.parser)
808 812 self.checker = IPDoctestOutputChecker()
809 813 self.globs = None
810 814 self.extraglobs = None
General Comments 0
You need to be logged in to leave comments. Login now