##// END OF EJS Templates
fix %autopx in scripts by calling run_code for each ast node...
MinRK -
Show More
@@ -1,2606 +1,2617 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 from __future__ import with_statement
18 18 from __future__ import absolute_import
19 19
20 20 import __builtin__
21 21 import __future__
22 22 import abc
23 23 import ast
24 24 import atexit
25 25 import codeop
26 26 import inspect
27 27 import os
28 28 import re
29 29 import sys
30 30 import tempfile
31 31 import types
32 32 from contextlib import nested
33 33
34 34 from IPython.config.configurable import Configurable
35 35 from IPython.core import debugger, oinspect
36 36 from IPython.core import history as ipcorehist
37 37 from IPython.core import page
38 38 from IPython.core import prefilter
39 39 from IPython.core import shadowns
40 40 from IPython.core import ultratb
41 41 from IPython.core.alias import AliasManager
42 42 from IPython.core.autocall import ExitAutocall
43 43 from IPython.core.builtin_trap import BuiltinTrap
44 44 from IPython.core.compilerop import CachingCompiler
45 45 from IPython.core.display_trap import DisplayTrap
46 46 from IPython.core.displayhook import DisplayHook
47 47 from IPython.core.displaypub import DisplayPublisher
48 48 from IPython.core.error import TryNext, UsageError
49 49 from IPython.core.extensions import ExtensionManager
50 50 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
51 51 from IPython.core.formatters import DisplayFormatter
52 52 from IPython.core.history import HistoryManager
53 53 from IPython.core.inputsplitter import IPythonInputSplitter
54 54 from IPython.core.logger import Logger
55 55 from IPython.core.macro import Macro
56 56 from IPython.core.magic import Magic
57 57 from IPython.core.payload import PayloadManager
58 58 from IPython.core.plugin import PluginManager
59 59 from IPython.core.prefilter import PrefilterManager, ESC_MAGIC
60 60 from IPython.external.Itpl import ItplNS
61 61 from IPython.utils import PyColorize
62 62 from IPython.utils import io
63 63 from IPython.utils.doctestreload import doctest_reload
64 64 from IPython.utils.io import ask_yes_no, rprint
65 65 from IPython.utils.ipstruct import Struct
66 66 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
67 67 from IPython.utils.pickleshare import PickleShareDB
68 68 from IPython.utils.process import system, getoutput
69 69 from IPython.utils.strdispatch import StrDispatch
70 70 from IPython.utils.syspathcontext import prepended_to_syspath
71 71 from IPython.utils.text import num_ini_spaces, format_screen, LSString, SList
72 72 from IPython.utils.traitlets import (Int, Str, CBool, CaselessStrEnum, Enum,
73 73 List, Unicode, Instance, Type)
74 74 from IPython.utils.warn import warn, error, fatal
75 75 import IPython.core.hooks
76 76
77 77 #-----------------------------------------------------------------------------
78 78 # Globals
79 79 #-----------------------------------------------------------------------------
80 80
81 81 # compiled regexps for autoindent management
82 82 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
83 83
84 84 #-----------------------------------------------------------------------------
85 85 # Utilities
86 86 #-----------------------------------------------------------------------------
87 87
88 88 # store the builtin raw_input globally, and use this always, in case user code
89 89 # overwrites it (like wx.py.PyShell does)
90 90 raw_input_original = raw_input
91 91
92 92 def softspace(file, newvalue):
93 93 """Copied from code.py, to remove the dependency"""
94 94
95 95 oldvalue = 0
96 96 try:
97 97 oldvalue = file.softspace
98 98 except AttributeError:
99 99 pass
100 100 try:
101 101 file.softspace = newvalue
102 102 except (AttributeError, TypeError):
103 103 # "attribute-less object" or "read-only attributes"
104 104 pass
105 105 return oldvalue
106 106
107 107
108 108 def no_op(*a, **kw): pass
109 109
110 110 class SpaceInInput(Exception): pass
111 111
112 112 class Bunch: pass
113 113
114 114
115 115 def get_default_colors():
116 116 if sys.platform=='darwin':
117 117 return "LightBG"
118 118 elif os.name=='nt':
119 119 return 'Linux'
120 120 else:
121 121 return 'Linux'
122 122
123 123
124 124 class SeparateStr(Str):
125 125 """A Str subclass to validate separate_in, separate_out, etc.
126 126
127 127 This is a Str based trait that converts '0'->'' and '\\n'->'\n'.
128 128 """
129 129
130 130 def validate(self, obj, value):
131 131 if value == '0': value = ''
132 132 value = value.replace('\\n','\n')
133 133 return super(SeparateStr, self).validate(obj, value)
134 134
135 135 class MultipleInstanceError(Exception):
136 136 pass
137 137
138 138 class ReadlineNoRecord(object):
139 139 """Context manager to execute some code, then reload readline history
140 140 so that interactive input to the code doesn't appear when pressing up."""
141 141 def __init__(self, shell):
142 142 self.shell = shell
143 143 self._nested_level = 0
144 144
145 145 def __enter__(self):
146 146 if self._nested_level == 0:
147 147 self.orig_length = self.current_length()
148 148 self.readline_tail = self.get_readline_tail()
149 149 self._nested_level += 1
150 150
151 151 def __exit__(self, type, value, traceback):
152 152 self._nested_level -= 1
153 153 if self._nested_level == 0:
154 154 # Try clipping the end if it's got longer
155 155 e = self.current_length() - self.orig_length
156 156 if e > 0:
157 157 for _ in range(e):
158 158 self.shell.readline.remove_history_item(self.orig_length)
159 159
160 160 # If it still doesn't match, just reload readline history.
161 161 if self.current_length() != self.orig_length \
162 162 or self.get_readline_tail() != self.readline_tail:
163 163 self.shell.refill_readline_hist()
164 164 # Returning False will cause exceptions to propagate
165 165 return False
166 166
167 167 def current_length(self):
168 168 return self.shell.readline.get_current_history_length()
169 169
170 170 def get_readline_tail(self, n=10):
171 171 """Get the last n items in readline history."""
172 172 end = self.shell.readline.get_current_history_length() + 1
173 173 start = max(end-n, 1)
174 174 ghi = self.shell.readline.get_history_item
175 175 return [ghi(x) for x in range(start, end)]
176 176
177 177
178 178 #-----------------------------------------------------------------------------
179 179 # Main IPython class
180 180 #-----------------------------------------------------------------------------
181 181
182 182 class InteractiveShell(Configurable, Magic):
183 183 """An enhanced, interactive shell for Python."""
184 184
185 185 _instance = None
186 186 autocall = Enum((0,1,2), default_value=1, config=True)
187 187 # TODO: remove all autoindent logic and put into frontends.
188 188 # We can't do this yet because even runlines uses the autoindent.
189 189 autoindent = CBool(True, config=True)
190 190 automagic = CBool(True, config=True)
191 191 cache_size = Int(1000, config=True)
192 192 color_info = CBool(True, config=True)
193 193 colors = CaselessStrEnum(('NoColor','LightBG','Linux'),
194 194 default_value=get_default_colors(), config=True)
195 195 debug = CBool(False, config=True)
196 196 deep_reload = CBool(False, config=True)
197 197 display_formatter = Instance(DisplayFormatter)
198 198 displayhook_class = Type(DisplayHook)
199 199 display_pub_class = Type(DisplayPublisher)
200 200
201 201 exit_now = CBool(False)
202 202 exiter = Instance(ExitAutocall)
203 203 def _exiter_default(self):
204 204 return ExitAutocall(self)
205 205 # Monotonically increasing execution counter
206 206 execution_count = Int(1)
207 207 filename = Unicode("<ipython console>")
208 208 ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
209 209
210 210 # Input splitter, to split entire cells of input into either individual
211 211 # interactive statements or whole blocks.
212 212 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
213 213 (), {})
214 214 logstart = CBool(False, config=True)
215 215 logfile = Unicode('', config=True)
216 216 logappend = Unicode('', config=True)
217 217 object_info_string_level = Enum((0,1,2), default_value=0,
218 218 config=True)
219 219 pdb = CBool(False, config=True)
220 220
221 221 profile = Unicode('', config=True)
222 222 prompt_in1 = Str('In [\\#]: ', config=True)
223 223 prompt_in2 = Str(' .\\D.: ', config=True)
224 224 prompt_out = Str('Out[\\#]: ', config=True)
225 225 prompts_pad_left = CBool(True, config=True)
226 226 quiet = CBool(False, config=True)
227 227
228 228 history_length = Int(10000, config=True)
229 229
230 230 # The readline stuff will eventually be moved to the terminal subclass
231 231 # but for now, we can't do that as readline is welded in everywhere.
232 232 readline_use = CBool(True, config=True)
233 233 readline_merge_completions = CBool(True, config=True)
234 234 readline_omit__names = Enum((0,1,2), default_value=2, config=True)
235 235 readline_remove_delims = Str('-/~', config=True)
236 236 readline_parse_and_bind = List([
237 237 'tab: complete',
238 238 '"\C-l": clear-screen',
239 239 'set show-all-if-ambiguous on',
240 240 '"\C-o": tab-insert',
241 241 # See bug gh-58 - with \M-i enabled, chars 0x9000-0x9fff
242 242 # crash IPython.
243 243 '"\M-o": "\d\d\d\d"',
244 244 '"\M-I": "\d\d\d\d"',
245 245 '"\C-r": reverse-search-history',
246 246 '"\C-s": forward-search-history',
247 247 '"\C-p": history-search-backward',
248 248 '"\C-n": history-search-forward',
249 249 '"\e[A": history-search-backward',
250 250 '"\e[B": history-search-forward',
251 251 '"\C-k": kill-line',
252 252 '"\C-u": unix-line-discard',
253 253 ], allow_none=False, config=True)
254 254
255 255 # TODO: this part of prompt management should be moved to the frontends.
256 256 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
257 257 separate_in = SeparateStr('\n', config=True)
258 258 separate_out = SeparateStr('', config=True)
259 259 separate_out2 = SeparateStr('', config=True)
260 260 wildcards_case_sensitive = CBool(True, config=True)
261 261 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
262 262 default_value='Context', config=True)
263 263
264 264 # Subcomponents of InteractiveShell
265 265 alias_manager = Instance('IPython.core.alias.AliasManager')
266 266 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager')
267 267 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap')
268 268 display_trap = Instance('IPython.core.display_trap.DisplayTrap')
269 269 extension_manager = Instance('IPython.core.extensions.ExtensionManager')
270 270 plugin_manager = Instance('IPython.core.plugin.PluginManager')
271 271 payload_manager = Instance('IPython.core.payload.PayloadManager')
272 272 history_manager = Instance('IPython.core.history.HistoryManager')
273 273
274 274 # Private interface
275 275 _post_execute = Instance(dict)
276 276
277 277 def __init__(self, config=None, ipython_dir=None,
278 278 user_ns=None, user_global_ns=None,
279 279 custom_exceptions=((), None)):
280 280
281 281 # This is where traits with a config_key argument are updated
282 282 # from the values on config.
283 283 super(InteractiveShell, self).__init__(config=config)
284 284
285 285 # These are relatively independent and stateless
286 286 self.init_ipython_dir(ipython_dir)
287 287 self.init_instance_attrs()
288 288 self.init_environment()
289 289
290 290 # Create namespaces (user_ns, user_global_ns, etc.)
291 291 self.init_create_namespaces(user_ns, user_global_ns)
292 292 # This has to be done after init_create_namespaces because it uses
293 293 # something in self.user_ns, but before init_sys_modules, which
294 294 # is the first thing to modify sys.
295 295 # TODO: When we override sys.stdout and sys.stderr before this class
296 296 # is created, we are saving the overridden ones here. Not sure if this
297 297 # is what we want to do.
298 298 self.save_sys_module_state()
299 299 self.init_sys_modules()
300 300
301 301 # While we're trying to have each part of the code directly access what
302 302 # it needs without keeping redundant references to objects, we have too
303 303 # much legacy code that expects ip.db to exist.
304 304 self.db = PickleShareDB(os.path.join(self.ipython_dir, 'db'))
305 305
306 306 self.init_history()
307 307 self.init_encoding()
308 308 self.init_prefilter()
309 309
310 310 Magic.__init__(self, self)
311 311
312 312 self.init_syntax_highlighting()
313 313 self.init_hooks()
314 314 self.init_pushd_popd_magic()
315 315 # self.init_traceback_handlers use to be here, but we moved it below
316 316 # because it and init_io have to come after init_readline.
317 317 self.init_user_ns()
318 318 self.init_logger()
319 319 self.init_alias()
320 320 self.init_builtins()
321 321
322 322 # pre_config_initialization
323 323
324 324 # The next section should contain everything that was in ipmaker.
325 325 self.init_logstart()
326 326
327 327 # The following was in post_config_initialization
328 328 self.init_inspector()
329 329 # init_readline() must come before init_io(), because init_io uses
330 330 # readline related things.
331 331 self.init_readline()
332 332 # init_completer must come after init_readline, because it needs to
333 333 # know whether readline is present or not system-wide to configure the
334 334 # completers, since the completion machinery can now operate
335 335 # independently of readline (e.g. over the network)
336 336 self.init_completer()
337 337 # TODO: init_io() needs to happen before init_traceback handlers
338 338 # because the traceback handlers hardcode the stdout/stderr streams.
339 339 # This logic in in debugger.Pdb and should eventually be changed.
340 340 self.init_io()
341 341 self.init_traceback_handlers(custom_exceptions)
342 342 self.init_prompts()
343 343 self.init_display_formatter()
344 344 self.init_display_pub()
345 345 self.init_displayhook()
346 346 self.init_reload_doctest()
347 347 self.init_magics()
348 348 self.init_pdb()
349 349 self.init_extension_manager()
350 350 self.init_plugin_manager()
351 351 self.init_payload()
352 352 self.hooks.late_startup_hook()
353 353 atexit.register(self.atexit_operations)
354 354
355 355 @classmethod
356 356 def instance(cls, *args, **kwargs):
357 357 """Returns a global InteractiveShell instance."""
358 358 if cls._instance is None:
359 359 inst = cls(*args, **kwargs)
360 360 # Now make sure that the instance will also be returned by
361 361 # the subclasses instance attribute.
362 362 for subclass in cls.mro():
363 363 if issubclass(cls, subclass) and \
364 364 issubclass(subclass, InteractiveShell):
365 365 subclass._instance = inst
366 366 else:
367 367 break
368 368 if isinstance(cls._instance, cls):
369 369 return cls._instance
370 370 else:
371 371 raise MultipleInstanceError(
372 372 'Multiple incompatible subclass instances of '
373 373 'InteractiveShell are being created.'
374 374 )
375 375
376 376 @classmethod
377 377 def initialized(cls):
378 378 return hasattr(cls, "_instance")
379 379
380 380 def get_ipython(self):
381 381 """Return the currently running IPython instance."""
382 382 return self
383 383
384 384 #-------------------------------------------------------------------------
385 385 # Trait changed handlers
386 386 #-------------------------------------------------------------------------
387 387
388 388 def _ipython_dir_changed(self, name, new):
389 389 if not os.path.isdir(new):
390 390 os.makedirs(new, mode = 0777)
391 391
392 392 def set_autoindent(self,value=None):
393 393 """Set the autoindent flag, checking for readline support.
394 394
395 395 If called with no arguments, it acts as a toggle."""
396 396
397 397 if not self.has_readline:
398 398 if os.name == 'posix':
399 399 warn("The auto-indent feature requires the readline library")
400 400 self.autoindent = 0
401 401 return
402 402 if value is None:
403 403 self.autoindent = not self.autoindent
404 404 else:
405 405 self.autoindent = value
406 406
407 407 #-------------------------------------------------------------------------
408 408 # init_* methods called by __init__
409 409 #-------------------------------------------------------------------------
410 410
411 411 def init_ipython_dir(self, ipython_dir):
412 412 if ipython_dir is not None:
413 413 self.ipython_dir = ipython_dir
414 414 self.config.Global.ipython_dir = self.ipython_dir
415 415 return
416 416
417 417 if hasattr(self.config.Global, 'ipython_dir'):
418 418 self.ipython_dir = self.config.Global.ipython_dir
419 419 else:
420 420 self.ipython_dir = get_ipython_dir()
421 421
422 422 # All children can just read this
423 423 self.config.Global.ipython_dir = self.ipython_dir
424 424
425 425 def init_instance_attrs(self):
426 426 self.more = False
427 427
428 428 # command compiler
429 429 self.compile = CachingCompiler()
430 430
431 431 # User input buffers
432 432 # NOTE: these variables are slated for full removal, once we are 100%
433 433 # sure that the new execution logic is solid. We will delte runlines,
434 434 # push_line and these buffers, as all input will be managed by the
435 435 # frontends via an inputsplitter instance.
436 436 self.buffer = []
437 437 self.buffer_raw = []
438 438
439 439 # Make an empty namespace, which extension writers can rely on both
440 440 # existing and NEVER being used by ipython itself. This gives them a
441 441 # convenient location for storing additional information and state
442 442 # their extensions may require, without fear of collisions with other
443 443 # ipython names that may develop later.
444 444 self.meta = Struct()
445 445
446 446 # Object variable to store code object waiting execution. This is
447 447 # used mainly by the multithreaded shells, but it can come in handy in
448 448 # other situations. No need to use a Queue here, since it's a single
449 449 # item which gets cleared once run.
450 450 self.code_to_run = None
451 451
452 452 # Temporary files used for various purposes. Deleted at exit.
453 453 self.tempfiles = []
454 454
455 455 # Keep track of readline usage (later set by init_readline)
456 456 self.has_readline = False
457 457
458 458 # keep track of where we started running (mainly for crash post-mortem)
459 459 # This is not being used anywhere currently.
460 460 self.starting_dir = os.getcwd()
461 461
462 462 # Indentation management
463 463 self.indent_current_nsp = 0
464 464
465 465 # Dict to track post-execution functions that have been registered
466 466 self._post_execute = {}
467 467
468 468 def init_environment(self):
469 469 """Any changes we need to make to the user's environment."""
470 470 pass
471 471
472 472 def init_encoding(self):
473 473 # Get system encoding at startup time. Certain terminals (like Emacs
474 474 # under Win32 have it set to None, and we need to have a known valid
475 475 # encoding to use in the raw_input() method
476 476 try:
477 477 self.stdin_encoding = sys.stdin.encoding or 'ascii'
478 478 except AttributeError:
479 479 self.stdin_encoding = 'ascii'
480 480
481 481 def init_syntax_highlighting(self):
482 482 # Python source parser/formatter for syntax highlighting
483 483 pyformat = PyColorize.Parser().format
484 484 self.pycolorize = lambda src: pyformat(src,'str',self.colors)
485 485
486 486 def init_pushd_popd_magic(self):
487 487 # for pushd/popd management
488 488 try:
489 489 self.home_dir = get_home_dir()
490 490 except HomeDirError, msg:
491 491 fatal(msg)
492 492
493 493 self.dir_stack = []
494 494
495 495 def init_logger(self):
496 496 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
497 497 logmode='rotate')
498 498
499 499 def init_logstart(self):
500 500 """Initialize logging in case it was requested at the command line.
501 501 """
502 502 if self.logappend:
503 503 self.magic_logstart(self.logappend + ' append')
504 504 elif self.logfile:
505 505 self.magic_logstart(self.logfile)
506 506 elif self.logstart:
507 507 self.magic_logstart()
508 508
509 509 def init_builtins(self):
510 510 self.builtin_trap = BuiltinTrap(shell=self)
511 511
512 512 def init_inspector(self):
513 513 # Object inspector
514 514 self.inspector = oinspect.Inspector(oinspect.InspectColors,
515 515 PyColorize.ANSICodeColors,
516 516 'NoColor',
517 517 self.object_info_string_level)
518 518
519 519 def init_io(self):
520 520 # This will just use sys.stdout and sys.stderr. If you want to
521 521 # override sys.stdout and sys.stderr themselves, you need to do that
522 522 # *before* instantiating this class, because Term holds onto
523 523 # references to the underlying streams.
524 524 if sys.platform == 'win32' and self.has_readline:
525 525 Term = io.IOTerm(cout=self.readline._outputfile,
526 526 cerr=self.readline._outputfile)
527 527 else:
528 528 Term = io.IOTerm()
529 529 io.Term = Term
530 530
531 531 def init_prompts(self):
532 532 # TODO: This is a pass for now because the prompts are managed inside
533 533 # the DisplayHook. Once there is a separate prompt manager, this
534 534 # will initialize that object and all prompt related information.
535 535 pass
536 536
537 537 def init_display_formatter(self):
538 538 self.display_formatter = DisplayFormatter(config=self.config)
539 539
540 540 def init_display_pub(self):
541 541 self.display_pub = self.display_pub_class(config=self.config)
542 542
543 543 def init_displayhook(self):
544 544 # Initialize displayhook, set in/out prompts and printing system
545 545 self.displayhook = self.displayhook_class(
546 546 config=self.config,
547 547 shell=self,
548 548 cache_size=self.cache_size,
549 549 input_sep = self.separate_in,
550 550 output_sep = self.separate_out,
551 551 output_sep2 = self.separate_out2,
552 552 ps1 = self.prompt_in1,
553 553 ps2 = self.prompt_in2,
554 554 ps_out = self.prompt_out,
555 555 pad_left = self.prompts_pad_left
556 556 )
557 557 # This is a context manager that installs/revmoes the displayhook at
558 558 # the appropriate time.
559 559 self.display_trap = DisplayTrap(hook=self.displayhook)
560 560
561 561 def init_reload_doctest(self):
562 562 # Do a proper resetting of doctest, including the necessary displayhook
563 563 # monkeypatching
564 564 try:
565 565 doctest_reload()
566 566 except ImportError:
567 567 warn("doctest module does not exist.")
568 568
569 569 #-------------------------------------------------------------------------
570 570 # Things related to injections into the sys module
571 571 #-------------------------------------------------------------------------
572 572
573 573 def save_sys_module_state(self):
574 574 """Save the state of hooks in the sys module.
575 575
576 576 This has to be called after self.user_ns is created.
577 577 """
578 578 self._orig_sys_module_state = {}
579 579 self._orig_sys_module_state['stdin'] = sys.stdin
580 580 self._orig_sys_module_state['stdout'] = sys.stdout
581 581 self._orig_sys_module_state['stderr'] = sys.stderr
582 582 self._orig_sys_module_state['excepthook'] = sys.excepthook
583 583 try:
584 584 self._orig_sys_modules_main_name = self.user_ns['__name__']
585 585 except KeyError:
586 586 pass
587 587
588 588 def restore_sys_module_state(self):
589 589 """Restore the state of the sys module."""
590 590 try:
591 591 for k, v in self._orig_sys_module_state.iteritems():
592 592 setattr(sys, k, v)
593 593 except AttributeError:
594 594 pass
595 595 # Reset what what done in self.init_sys_modules
596 596 try:
597 597 sys.modules[self.user_ns['__name__']] = self._orig_sys_modules_main_name
598 598 except (AttributeError, KeyError):
599 599 pass
600 600
601 601 #-------------------------------------------------------------------------
602 602 # Things related to hooks
603 603 #-------------------------------------------------------------------------
604 604
605 605 def init_hooks(self):
606 606 # hooks holds pointers used for user-side customizations
607 607 self.hooks = Struct()
608 608
609 609 self.strdispatchers = {}
610 610
611 611 # Set all default hooks, defined in the IPython.hooks module.
612 612 hooks = IPython.core.hooks
613 613 for hook_name in hooks.__all__:
614 614 # default hooks have priority 100, i.e. low; user hooks should have
615 615 # 0-100 priority
616 616 self.set_hook(hook_name,getattr(hooks,hook_name), 100)
617 617
618 618 def set_hook(self,name,hook, priority = 50, str_key = None, re_key = None):
619 619 """set_hook(name,hook) -> sets an internal IPython hook.
620 620
621 621 IPython exposes some of its internal API as user-modifiable hooks. By
622 622 adding your function to one of these hooks, you can modify IPython's
623 623 behavior to call at runtime your own routines."""
624 624
625 625 # At some point in the future, this should validate the hook before it
626 626 # accepts it. Probably at least check that the hook takes the number
627 627 # of args it's supposed to.
628 628
629 629 f = types.MethodType(hook,self)
630 630
631 631 # check if the hook is for strdispatcher first
632 632 if str_key is not None:
633 633 sdp = self.strdispatchers.get(name, StrDispatch())
634 634 sdp.add_s(str_key, f, priority )
635 635 self.strdispatchers[name] = sdp
636 636 return
637 637 if re_key is not None:
638 638 sdp = self.strdispatchers.get(name, StrDispatch())
639 639 sdp.add_re(re.compile(re_key), f, priority )
640 640 self.strdispatchers[name] = sdp
641 641 return
642 642
643 643 dp = getattr(self.hooks, name, None)
644 644 if name not in IPython.core.hooks.__all__:
645 645 print "Warning! Hook '%s' is not one of %s" % \
646 646 (name, IPython.core.hooks.__all__ )
647 647 if not dp:
648 648 dp = IPython.core.hooks.CommandChainDispatcher()
649 649
650 650 try:
651 651 dp.add(f,priority)
652 652 except AttributeError:
653 653 # it was not commandchain, plain old func - replace
654 654 dp = f
655 655
656 656 setattr(self.hooks,name, dp)
657 657
658 658 def register_post_execute(self, func):
659 659 """Register a function for calling after code execution.
660 660 """
661 661 if not callable(func):
662 662 raise ValueError('argument %s must be callable' % func)
663 663 self._post_execute[func] = True
664 664
665 665 #-------------------------------------------------------------------------
666 666 # Things related to the "main" module
667 667 #-------------------------------------------------------------------------
668 668
669 669 def new_main_mod(self,ns=None):
670 670 """Return a new 'main' module object for user code execution.
671 671 """
672 672 main_mod = self._user_main_module
673 673 init_fakemod_dict(main_mod,ns)
674 674 return main_mod
675 675
676 676 def cache_main_mod(self,ns,fname):
677 677 """Cache a main module's namespace.
678 678
679 679 When scripts are executed via %run, we must keep a reference to the
680 680 namespace of their __main__ module (a FakeModule instance) around so
681 681 that Python doesn't clear it, rendering objects defined therein
682 682 useless.
683 683
684 684 This method keeps said reference in a private dict, keyed by the
685 685 absolute path of the module object (which corresponds to the script
686 686 path). This way, for multiple executions of the same script we only
687 687 keep one copy of the namespace (the last one), thus preventing memory
688 688 leaks from old references while allowing the objects from the last
689 689 execution to be accessible.
690 690
691 691 Note: we can not allow the actual FakeModule instances to be deleted,
692 692 because of how Python tears down modules (it hard-sets all their
693 693 references to None without regard for reference counts). This method
694 694 must therefore make a *copy* of the given namespace, to allow the
695 695 original module's __dict__ to be cleared and reused.
696 696
697 697
698 698 Parameters
699 699 ----------
700 700 ns : a namespace (a dict, typically)
701 701
702 702 fname : str
703 703 Filename associated with the namespace.
704 704
705 705 Examples
706 706 --------
707 707
708 708 In [10]: import IPython
709 709
710 710 In [11]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
711 711
712 712 In [12]: IPython.__file__ in _ip._main_ns_cache
713 713 Out[12]: True
714 714 """
715 715 self._main_ns_cache[os.path.abspath(fname)] = ns.copy()
716 716
717 717 def clear_main_mod_cache(self):
718 718 """Clear the cache of main modules.
719 719
720 720 Mainly for use by utilities like %reset.
721 721
722 722 Examples
723 723 --------
724 724
725 725 In [15]: import IPython
726 726
727 727 In [16]: _ip.cache_main_mod(IPython.__dict__,IPython.__file__)
728 728
729 729 In [17]: len(_ip._main_ns_cache) > 0
730 730 Out[17]: True
731 731
732 732 In [18]: _ip.clear_main_mod_cache()
733 733
734 734 In [19]: len(_ip._main_ns_cache) == 0
735 735 Out[19]: True
736 736 """
737 737 self._main_ns_cache.clear()
738 738
739 739 #-------------------------------------------------------------------------
740 740 # Things related to debugging
741 741 #-------------------------------------------------------------------------
742 742
743 743 def init_pdb(self):
744 744 # Set calling of pdb on exceptions
745 745 # self.call_pdb is a property
746 746 self.call_pdb = self.pdb
747 747
748 748 def _get_call_pdb(self):
749 749 return self._call_pdb
750 750
751 751 def _set_call_pdb(self,val):
752 752
753 753 if val not in (0,1,False,True):
754 754 raise ValueError,'new call_pdb value must be boolean'
755 755
756 756 # store value in instance
757 757 self._call_pdb = val
758 758
759 759 # notify the actual exception handlers
760 760 self.InteractiveTB.call_pdb = val
761 761
762 762 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
763 763 'Control auto-activation of pdb at exceptions')
764 764
765 765 def debugger(self,force=False):
766 766 """Call the pydb/pdb debugger.
767 767
768 768 Keywords:
769 769
770 770 - force(False): by default, this routine checks the instance call_pdb
771 771 flag and does not actually invoke the debugger if the flag is false.
772 772 The 'force' option forces the debugger to activate even if the flag
773 773 is false.
774 774 """
775 775
776 776 if not (force or self.call_pdb):
777 777 return
778 778
779 779 if not hasattr(sys,'last_traceback'):
780 780 error('No traceback has been produced, nothing to debug.')
781 781 return
782 782
783 783 # use pydb if available
784 784 if debugger.has_pydb:
785 785 from pydb import pm
786 786 else:
787 787 # fallback to our internal debugger
788 788 pm = lambda : self.InteractiveTB.debugger(force=True)
789 789
790 790 with self.readline_no_record:
791 791 pm()
792 792
793 793 #-------------------------------------------------------------------------
794 794 # Things related to IPython's various namespaces
795 795 #-------------------------------------------------------------------------
796 796
797 797 def init_create_namespaces(self, user_ns=None, user_global_ns=None):
798 798 # Create the namespace where the user will operate. user_ns is
799 799 # normally the only one used, and it is passed to the exec calls as
800 800 # the locals argument. But we do carry a user_global_ns namespace
801 801 # given as the exec 'globals' argument, This is useful in embedding
802 802 # situations where the ipython shell opens in a context where the
803 803 # distinction between locals and globals is meaningful. For
804 804 # non-embedded contexts, it is just the same object as the user_ns dict.
805 805
806 806 # FIXME. For some strange reason, __builtins__ is showing up at user
807 807 # level as a dict instead of a module. This is a manual fix, but I
808 808 # should really track down where the problem is coming from. Alex
809 809 # Schmolck reported this problem first.
810 810
811 811 # A useful post by Alex Martelli on this topic:
812 812 # Re: inconsistent value from __builtins__
813 813 # Von: Alex Martelli <aleaxit@yahoo.com>
814 814 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
815 815 # Gruppen: comp.lang.python
816 816
817 817 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
818 818 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
819 819 # > <type 'dict'>
820 820 # > >>> print type(__builtins__)
821 821 # > <type 'module'>
822 822 # > Is this difference in return value intentional?
823 823
824 824 # Well, it's documented that '__builtins__' can be either a dictionary
825 825 # or a module, and it's been that way for a long time. Whether it's
826 826 # intentional (or sensible), I don't know. In any case, the idea is
827 827 # that if you need to access the built-in namespace directly, you
828 828 # should start with "import __builtin__" (note, no 's') which will
829 829 # definitely give you a module. Yeah, it's somewhat confusing:-(.
830 830
831 831 # These routines return properly built dicts as needed by the rest of
832 832 # the code, and can also be used by extension writers to generate
833 833 # properly initialized namespaces.
834 834 user_ns, user_global_ns = self.make_user_namespaces(user_ns,
835 835 user_global_ns)
836 836
837 837 # Assign namespaces
838 838 # This is the namespace where all normal user variables live
839 839 self.user_ns = user_ns
840 840 self.user_global_ns = user_global_ns
841 841
842 842 # An auxiliary namespace that checks what parts of the user_ns were
843 843 # loaded at startup, so we can list later only variables defined in
844 844 # actual interactive use. Since it is always a subset of user_ns, it
845 845 # doesn't need to be separately tracked in the ns_table.
846 846 self.user_ns_hidden = {}
847 847
848 848 # A namespace to keep track of internal data structures to prevent
849 849 # them from cluttering user-visible stuff. Will be updated later
850 850 self.internal_ns = {}
851 851
852 852 # Now that FakeModule produces a real module, we've run into a nasty
853 853 # problem: after script execution (via %run), the module where the user
854 854 # code ran is deleted. Now that this object is a true module (needed
855 855 # so docetst and other tools work correctly), the Python module
856 856 # teardown mechanism runs over it, and sets to None every variable
857 857 # present in that module. Top-level references to objects from the
858 858 # script survive, because the user_ns is updated with them. However,
859 859 # calling functions defined in the script that use other things from
860 860 # the script will fail, because the function's closure had references
861 861 # to the original objects, which are now all None. So we must protect
862 862 # these modules from deletion by keeping a cache.
863 863 #
864 864 # To avoid keeping stale modules around (we only need the one from the
865 865 # last run), we use a dict keyed with the full path to the script, so
866 866 # only the last version of the module is held in the cache. Note,
867 867 # however, that we must cache the module *namespace contents* (their
868 868 # __dict__). Because if we try to cache the actual modules, old ones
869 869 # (uncached) could be destroyed while still holding references (such as
870 870 # those held by GUI objects that tend to be long-lived)>
871 871 #
872 872 # The %reset command will flush this cache. See the cache_main_mod()
873 873 # and clear_main_mod_cache() methods for details on use.
874 874
875 875 # This is the cache used for 'main' namespaces
876 876 self._main_ns_cache = {}
877 877 # And this is the single instance of FakeModule whose __dict__ we keep
878 878 # copying and clearing for reuse on each %run
879 879 self._user_main_module = FakeModule()
880 880
881 881 # A table holding all the namespaces IPython deals with, so that
882 882 # introspection facilities can search easily.
883 883 self.ns_table = {'user':user_ns,
884 884 'user_global':user_global_ns,
885 885 'internal':self.internal_ns,
886 886 'builtin':__builtin__.__dict__
887 887 }
888 888
889 889 # Similarly, track all namespaces where references can be held and that
890 890 # we can safely clear (so it can NOT include builtin). This one can be
891 891 # a simple list. Note that the main execution namespaces, user_ns and
892 892 # user_global_ns, can NOT be listed here, as clearing them blindly
893 893 # causes errors in object __del__ methods. Instead, the reset() method
894 894 # clears them manually and carefully.
895 895 self.ns_refs_table = [ self.user_ns_hidden,
896 896 self.internal_ns, self._main_ns_cache ]
897 897
898 898 def make_user_namespaces(self, user_ns=None, user_global_ns=None):
899 899 """Return a valid local and global user interactive namespaces.
900 900
901 901 This builds a dict with the minimal information needed to operate as a
902 902 valid IPython user namespace, which you can pass to the various
903 903 embedding classes in ipython. The default implementation returns the
904 904 same dict for both the locals and the globals to allow functions to
905 905 refer to variables in the namespace. Customized implementations can
906 906 return different dicts. The locals dictionary can actually be anything
907 907 following the basic mapping protocol of a dict, but the globals dict
908 908 must be a true dict, not even a subclass. It is recommended that any
909 909 custom object for the locals namespace synchronize with the globals
910 910 dict somehow.
911 911
912 912 Raises TypeError if the provided globals namespace is not a true dict.
913 913
914 914 Parameters
915 915 ----------
916 916 user_ns : dict-like, optional
917 917 The current user namespace. The items in this namespace should
918 918 be included in the output. If None, an appropriate blank
919 919 namespace should be created.
920 920 user_global_ns : dict, optional
921 921 The current user global namespace. The items in this namespace
922 922 should be included in the output. If None, an appropriate
923 923 blank namespace should be created.
924 924
925 925 Returns
926 926 -------
927 927 A pair of dictionary-like object to be used as the local namespace
928 928 of the interpreter and a dict to be used as the global namespace.
929 929 """
930 930
931 931
932 932 # We must ensure that __builtin__ (without the final 's') is always
933 933 # available and pointing to the __builtin__ *module*. For more details:
934 934 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
935 935
936 936 if user_ns is None:
937 937 # Set __name__ to __main__ to better match the behavior of the
938 938 # normal interpreter.
939 939 user_ns = {'__name__' :'__main__',
940 940 '__builtin__' : __builtin__,
941 941 '__builtins__' : __builtin__,
942 942 }
943 943 else:
944 944 user_ns.setdefault('__name__','__main__')
945 945 user_ns.setdefault('__builtin__',__builtin__)
946 946 user_ns.setdefault('__builtins__',__builtin__)
947 947
948 948 if user_global_ns is None:
949 949 user_global_ns = user_ns
950 950 if type(user_global_ns) is not dict:
951 951 raise TypeError("user_global_ns must be a true dict; got %r"
952 952 % type(user_global_ns))
953 953
954 954 return user_ns, user_global_ns
955 955
956 956 def init_sys_modules(self):
957 957 # We need to insert into sys.modules something that looks like a
958 958 # module but which accesses the IPython namespace, for shelve and
959 959 # pickle to work interactively. Normally they rely on getting
960 960 # everything out of __main__, but for embedding purposes each IPython
961 961 # instance has its own private namespace, so we can't go shoving
962 962 # everything into __main__.
963 963
964 964 # note, however, that we should only do this for non-embedded
965 965 # ipythons, which really mimic the __main__.__dict__ with their own
966 966 # namespace. Embedded instances, on the other hand, should not do
967 967 # this because they need to manage the user local/global namespaces
968 968 # only, but they live within a 'normal' __main__ (meaning, they
969 969 # shouldn't overtake the execution environment of the script they're
970 970 # embedded in).
971 971
972 972 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
973 973
974 974 try:
975 975 main_name = self.user_ns['__name__']
976 976 except KeyError:
977 977 raise KeyError('user_ns dictionary MUST have a "__name__" key')
978 978 else:
979 979 sys.modules[main_name] = FakeModule(self.user_ns)
980 980
981 981 def init_user_ns(self):
982 982 """Initialize all user-visible namespaces to their minimum defaults.
983 983
984 984 Certain history lists are also initialized here, as they effectively
985 985 act as user namespaces.
986 986
987 987 Notes
988 988 -----
989 989 All data structures here are only filled in, they are NOT reset by this
990 990 method. If they were not empty before, data will simply be added to
991 991 therm.
992 992 """
993 993 # This function works in two parts: first we put a few things in
994 994 # user_ns, and we sync that contents into user_ns_hidden so that these
995 995 # initial variables aren't shown by %who. After the sync, we add the
996 996 # rest of what we *do* want the user to see with %who even on a new
997 997 # session (probably nothing, so theye really only see their own stuff)
998 998
999 999 # The user dict must *always* have a __builtin__ reference to the
1000 1000 # Python standard __builtin__ namespace, which must be imported.
1001 1001 # This is so that certain operations in prompt evaluation can be
1002 1002 # reliably executed with builtins. Note that we can NOT use
1003 1003 # __builtins__ (note the 's'), because that can either be a dict or a
1004 1004 # module, and can even mutate at runtime, depending on the context
1005 1005 # (Python makes no guarantees on it). In contrast, __builtin__ is
1006 1006 # always a module object, though it must be explicitly imported.
1007 1007
1008 1008 # For more details:
1009 1009 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1010 1010 ns = dict(__builtin__ = __builtin__)
1011 1011
1012 1012 # Put 'help' in the user namespace
1013 1013 try:
1014 1014 from site import _Helper
1015 1015 ns['help'] = _Helper()
1016 1016 except ImportError:
1017 1017 warn('help() not available - check site.py')
1018 1018
1019 1019 # make global variables for user access to the histories
1020 1020 ns['_ih'] = self.history_manager.input_hist_parsed
1021 1021 ns['_oh'] = self.history_manager.output_hist
1022 1022 ns['_dh'] = self.history_manager.dir_hist
1023 1023
1024 1024 ns['_sh'] = shadowns
1025 1025
1026 1026 # user aliases to input and output histories. These shouldn't show up
1027 1027 # in %who, as they can have very large reprs.
1028 1028 ns['In'] = self.history_manager.input_hist_parsed
1029 1029 ns['Out'] = self.history_manager.output_hist
1030 1030
1031 1031 # Store myself as the public api!!!
1032 1032 ns['get_ipython'] = self.get_ipython
1033 1033
1034 1034 ns['exit'] = self.exiter
1035 1035 ns['quit'] = self.exiter
1036 1036
1037 1037 # Sync what we've added so far to user_ns_hidden so these aren't seen
1038 1038 # by %who
1039 1039 self.user_ns_hidden.update(ns)
1040 1040
1041 1041 # Anything put into ns now would show up in %who. Think twice before
1042 1042 # putting anything here, as we really want %who to show the user their
1043 1043 # stuff, not our variables.
1044 1044
1045 1045 # Finally, update the real user's namespace
1046 1046 self.user_ns.update(ns)
1047 1047
1048 1048 def reset(self, new_session=True):
1049 1049 """Clear all internal namespaces.
1050 1050
1051 1051 Note that this is much more aggressive than %reset, since it clears
1052 1052 fully all namespaces, as well as all input/output lists.
1053 1053
1054 1054 If new_session is True, a new history session will be opened.
1055 1055 """
1056 1056 # Clear histories
1057 1057 self.history_manager.reset(new_session)
1058 1058 # Reset counter used to index all histories
1059 1059 if new_session:
1060 1060 self.execution_count = 1
1061 1061
1062 1062 # Flush cached output items
1063 1063 self.displayhook.flush()
1064 1064
1065 1065 # Restore the user namespaces to minimal usability
1066 1066 for ns in self.ns_refs_table:
1067 1067 ns.clear()
1068 1068
1069 1069 # The main execution namespaces must be cleared very carefully,
1070 1070 # skipping the deletion of the builtin-related keys, because doing so
1071 1071 # would cause errors in many object's __del__ methods.
1072 1072 for ns in [self.user_ns, self.user_global_ns]:
1073 1073 drop_keys = set(ns.keys())
1074 1074 drop_keys.discard('__builtin__')
1075 1075 drop_keys.discard('__builtins__')
1076 1076 for k in drop_keys:
1077 1077 del ns[k]
1078 1078
1079 1079 # Restore the user namespaces to minimal usability
1080 1080 self.init_user_ns()
1081 1081
1082 1082 # Restore the default and user aliases
1083 1083 self.alias_manager.clear_aliases()
1084 1084 self.alias_manager.init_aliases()
1085 1085
1086 1086 # Flush the private list of module references kept for script
1087 1087 # execution protection
1088 1088 self.clear_main_mod_cache()
1089 1089
1090 1090 def reset_selective(self, regex=None):
1091 1091 """Clear selective variables from internal namespaces based on a
1092 1092 specified regular expression.
1093 1093
1094 1094 Parameters
1095 1095 ----------
1096 1096 regex : string or compiled pattern, optional
1097 1097 A regular expression pattern that will be used in searching
1098 1098 variable names in the users namespaces.
1099 1099 """
1100 1100 if regex is not None:
1101 1101 try:
1102 1102 m = re.compile(regex)
1103 1103 except TypeError:
1104 1104 raise TypeError('regex must be a string or compiled pattern')
1105 1105 # Search for keys in each namespace that match the given regex
1106 1106 # If a match is found, delete the key/value pair.
1107 1107 for ns in self.ns_refs_table:
1108 1108 for var in ns:
1109 1109 if m.search(var):
1110 1110 del ns[var]
1111 1111
1112 1112 def push(self, variables, interactive=True):
1113 1113 """Inject a group of variables into the IPython user namespace.
1114 1114
1115 1115 Parameters
1116 1116 ----------
1117 1117 variables : dict, str or list/tuple of str
1118 1118 The variables to inject into the user's namespace. If a dict, a
1119 1119 simple update is done. If a str, the string is assumed to have
1120 1120 variable names separated by spaces. A list/tuple of str can also
1121 1121 be used to give the variable names. If just the variable names are
1122 1122 give (list/tuple/str) then the variable values looked up in the
1123 1123 callers frame.
1124 1124 interactive : bool
1125 1125 If True (default), the variables will be listed with the ``who``
1126 1126 magic.
1127 1127 """
1128 1128 vdict = None
1129 1129
1130 1130 # We need a dict of name/value pairs to do namespace updates.
1131 1131 if isinstance(variables, dict):
1132 1132 vdict = variables
1133 1133 elif isinstance(variables, (basestring, list, tuple)):
1134 1134 if isinstance(variables, basestring):
1135 1135 vlist = variables.split()
1136 1136 else:
1137 1137 vlist = variables
1138 1138 vdict = {}
1139 1139 cf = sys._getframe(1)
1140 1140 for name in vlist:
1141 1141 try:
1142 1142 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1143 1143 except:
1144 1144 print ('Could not get variable %s from %s' %
1145 1145 (name,cf.f_code.co_name))
1146 1146 else:
1147 1147 raise ValueError('variables must be a dict/str/list/tuple')
1148 1148
1149 1149 # Propagate variables to user namespace
1150 1150 self.user_ns.update(vdict)
1151 1151
1152 1152 # And configure interactive visibility
1153 1153 config_ns = self.user_ns_hidden
1154 1154 if interactive:
1155 1155 for name, val in vdict.iteritems():
1156 1156 config_ns.pop(name, None)
1157 1157 else:
1158 1158 for name,val in vdict.iteritems():
1159 1159 config_ns[name] = val
1160 1160
1161 1161 #-------------------------------------------------------------------------
1162 1162 # Things related to object introspection
1163 1163 #-------------------------------------------------------------------------
1164 1164
1165 1165 def _ofind(self, oname, namespaces=None):
1166 1166 """Find an object in the available namespaces.
1167 1167
1168 1168 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1169 1169
1170 1170 Has special code to detect magic functions.
1171 1171 """
1172 1172 #oname = oname.strip()
1173 1173 #print '1- oname: <%r>' % oname # dbg
1174 1174 try:
1175 1175 oname = oname.strip().encode('ascii')
1176 1176 #print '2- oname: <%r>' % oname # dbg
1177 1177 except UnicodeEncodeError:
1178 1178 print 'Python identifiers can only contain ascii characters.'
1179 1179 return dict(found=False)
1180 1180
1181 1181 alias_ns = None
1182 1182 if namespaces is None:
1183 1183 # Namespaces to search in:
1184 1184 # Put them in a list. The order is important so that we
1185 1185 # find things in the same order that Python finds them.
1186 1186 namespaces = [ ('Interactive', self.user_ns),
1187 1187 ('IPython internal', self.internal_ns),
1188 1188 ('Python builtin', __builtin__.__dict__),
1189 1189 ('Alias', self.alias_manager.alias_table),
1190 1190 ]
1191 1191 alias_ns = self.alias_manager.alias_table
1192 1192
1193 1193 # initialize results to 'null'
1194 1194 found = False; obj = None; ospace = None; ds = None;
1195 1195 ismagic = False; isalias = False; parent = None
1196 1196
1197 1197 # We need to special-case 'print', which as of python2.6 registers as a
1198 1198 # function but should only be treated as one if print_function was
1199 1199 # loaded with a future import. In this case, just bail.
1200 1200 if (oname == 'print' and not (self.compile.compiler_flags &
1201 1201 __future__.CO_FUTURE_PRINT_FUNCTION)):
1202 1202 return {'found':found, 'obj':obj, 'namespace':ospace,
1203 1203 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1204 1204
1205 1205 # Look for the given name by splitting it in parts. If the head is
1206 1206 # found, then we look for all the remaining parts as members, and only
1207 1207 # declare success if we can find them all.
1208 1208 oname_parts = oname.split('.')
1209 1209 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1210 1210 for nsname,ns in namespaces:
1211 1211 try:
1212 1212 obj = ns[oname_head]
1213 1213 except KeyError:
1214 1214 continue
1215 1215 else:
1216 1216 #print 'oname_rest:', oname_rest # dbg
1217 1217 for part in oname_rest:
1218 1218 try:
1219 1219 parent = obj
1220 1220 obj = getattr(obj,part)
1221 1221 except:
1222 1222 # Blanket except b/c some badly implemented objects
1223 1223 # allow __getattr__ to raise exceptions other than
1224 1224 # AttributeError, which then crashes IPython.
1225 1225 break
1226 1226 else:
1227 1227 # If we finish the for loop (no break), we got all members
1228 1228 found = True
1229 1229 ospace = nsname
1230 1230 if ns == alias_ns:
1231 1231 isalias = True
1232 1232 break # namespace loop
1233 1233
1234 1234 # Try to see if it's magic
1235 1235 if not found:
1236 1236 if oname.startswith(ESC_MAGIC):
1237 1237 oname = oname[1:]
1238 1238 obj = getattr(self,'magic_'+oname,None)
1239 1239 if obj is not None:
1240 1240 found = True
1241 1241 ospace = 'IPython internal'
1242 1242 ismagic = True
1243 1243
1244 1244 # Last try: special-case some literals like '', [], {}, etc:
1245 1245 if not found and oname_head in ["''",'""','[]','{}','()']:
1246 1246 obj = eval(oname_head)
1247 1247 found = True
1248 1248 ospace = 'Interactive'
1249 1249
1250 1250 return {'found':found, 'obj':obj, 'namespace':ospace,
1251 1251 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1252 1252
1253 1253 def _ofind_property(self, oname, info):
1254 1254 """Second part of object finding, to look for property details."""
1255 1255 if info.found:
1256 1256 # Get the docstring of the class property if it exists.
1257 1257 path = oname.split('.')
1258 1258 root = '.'.join(path[:-1])
1259 1259 if info.parent is not None:
1260 1260 try:
1261 1261 target = getattr(info.parent, '__class__')
1262 1262 # The object belongs to a class instance.
1263 1263 try:
1264 1264 target = getattr(target, path[-1])
1265 1265 # The class defines the object.
1266 1266 if isinstance(target, property):
1267 1267 oname = root + '.__class__.' + path[-1]
1268 1268 info = Struct(self._ofind(oname))
1269 1269 except AttributeError: pass
1270 1270 except AttributeError: pass
1271 1271
1272 1272 # We return either the new info or the unmodified input if the object
1273 1273 # hadn't been found
1274 1274 return info
1275 1275
1276 1276 def _object_find(self, oname, namespaces=None):
1277 1277 """Find an object and return a struct with info about it."""
1278 1278 inf = Struct(self._ofind(oname, namespaces))
1279 1279 return Struct(self._ofind_property(oname, inf))
1280 1280
1281 1281 def _inspect(self, meth, oname, namespaces=None, **kw):
1282 1282 """Generic interface to the inspector system.
1283 1283
1284 1284 This function is meant to be called by pdef, pdoc & friends."""
1285 1285 info = self._object_find(oname)
1286 1286 if info.found:
1287 1287 pmethod = getattr(self.inspector, meth)
1288 1288 formatter = format_screen if info.ismagic else None
1289 1289 if meth == 'pdoc':
1290 1290 pmethod(info.obj, oname, formatter)
1291 1291 elif meth == 'pinfo':
1292 1292 pmethod(info.obj, oname, formatter, info, **kw)
1293 1293 else:
1294 1294 pmethod(info.obj, oname)
1295 1295 else:
1296 1296 print 'Object `%s` not found.' % oname
1297 1297 return 'not found' # so callers can take other action
1298 1298
1299 1299 def object_inspect(self, oname):
1300 1300 with self.builtin_trap:
1301 1301 info = self._object_find(oname)
1302 1302 if info.found:
1303 1303 return self.inspector.info(info.obj, oname, info=info)
1304 1304 else:
1305 1305 return oinspect.object_info(name=oname, found=False)
1306 1306
1307 1307 #-------------------------------------------------------------------------
1308 1308 # Things related to history management
1309 1309 #-------------------------------------------------------------------------
1310 1310
1311 1311 def init_history(self):
1312 1312 """Sets up the command history, and starts regular autosaves."""
1313 1313 self.history_manager = HistoryManager(shell=self, config=self.config)
1314 1314
1315 1315 #-------------------------------------------------------------------------
1316 1316 # Things related to exception handling and tracebacks (not debugging)
1317 1317 #-------------------------------------------------------------------------
1318 1318
1319 1319 def init_traceback_handlers(self, custom_exceptions):
1320 1320 # Syntax error handler.
1321 1321 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor')
1322 1322
1323 1323 # The interactive one is initialized with an offset, meaning we always
1324 1324 # want to remove the topmost item in the traceback, which is our own
1325 1325 # internal code. Valid modes: ['Plain','Context','Verbose']
1326 1326 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1327 1327 color_scheme='NoColor',
1328 1328 tb_offset = 1,
1329 1329 check_cache=self.compile.check_cache)
1330 1330
1331 1331 # The instance will store a pointer to the system-wide exception hook,
1332 1332 # so that runtime code (such as magics) can access it. This is because
1333 1333 # during the read-eval loop, it may get temporarily overwritten.
1334 1334 self.sys_excepthook = sys.excepthook
1335 1335
1336 1336 # and add any custom exception handlers the user may have specified
1337 1337 self.set_custom_exc(*custom_exceptions)
1338 1338
1339 1339 # Set the exception mode
1340 1340 self.InteractiveTB.set_mode(mode=self.xmode)
1341 1341
1342 1342 def set_custom_exc(self, exc_tuple, handler):
1343 1343 """set_custom_exc(exc_tuple,handler)
1344 1344
1345 1345 Set a custom exception handler, which will be called if any of the
1346 1346 exceptions in exc_tuple occur in the mainloop (specifically, in the
1347 1347 run_code() method.
1348 1348
1349 1349 Inputs:
1350 1350
1351 1351 - exc_tuple: a *tuple* of valid exceptions to call the defined
1352 1352 handler for. It is very important that you use a tuple, and NOT A
1353 1353 LIST here, because of the way Python's except statement works. If
1354 1354 you only want to trap a single exception, use a singleton tuple:
1355 1355
1356 1356 exc_tuple == (MyCustomException,)
1357 1357
1358 1358 - handler: this must be defined as a function with the following
1359 1359 basic interface::
1360 1360
1361 1361 def my_handler(self, etype, value, tb, tb_offset=None)
1362 1362 ...
1363 1363 # The return value must be
1364 1364 return structured_traceback
1365 1365
1366 1366 This will be made into an instance method (via types.MethodType)
1367 1367 of IPython itself, and it will be called if any of the exceptions
1368 1368 listed in the exc_tuple are caught. If the handler is None, an
1369 1369 internal basic one is used, which just prints basic info.
1370 1370
1371 1371 WARNING: by putting in your own exception handler into IPython's main
1372 1372 execution loop, you run a very good chance of nasty crashes. This
1373 1373 facility should only be used if you really know what you are doing."""
1374 1374
1375 1375 assert type(exc_tuple)==type(()) , \
1376 1376 "The custom exceptions must be given AS A TUPLE."
1377 1377
1378 1378 def dummy_handler(self,etype,value,tb):
1379 1379 print '*** Simple custom exception handler ***'
1380 1380 print 'Exception type :',etype
1381 1381 print 'Exception value:',value
1382 1382 print 'Traceback :',tb
1383 1383 print 'Source code :','\n'.join(self.buffer)
1384 1384
1385 1385 if handler is None: handler = dummy_handler
1386 1386
1387 1387 self.CustomTB = types.MethodType(handler,self)
1388 1388 self.custom_exceptions = exc_tuple
1389 1389
1390 1390 def excepthook(self, etype, value, tb):
1391 1391 """One more defense for GUI apps that call sys.excepthook.
1392 1392
1393 1393 GUI frameworks like wxPython trap exceptions and call
1394 1394 sys.excepthook themselves. I guess this is a feature that
1395 1395 enables them to keep running after exceptions that would
1396 1396 otherwise kill their mainloop. This is a bother for IPython
1397 1397 which excepts to catch all of the program exceptions with a try:
1398 1398 except: statement.
1399 1399
1400 1400 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1401 1401 any app directly invokes sys.excepthook, it will look to the user like
1402 1402 IPython crashed. In order to work around this, we can disable the
1403 1403 CrashHandler and replace it with this excepthook instead, which prints a
1404 1404 regular traceback using our InteractiveTB. In this fashion, apps which
1405 1405 call sys.excepthook will generate a regular-looking exception from
1406 1406 IPython, and the CrashHandler will only be triggered by real IPython
1407 1407 crashes.
1408 1408
1409 1409 This hook should be used sparingly, only in places which are not likely
1410 1410 to be true IPython errors.
1411 1411 """
1412 1412 self.showtraceback((etype,value,tb),tb_offset=0)
1413 1413
1414 1414 def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None,
1415 1415 exception_only=False):
1416 1416 """Display the exception that just occurred.
1417 1417
1418 1418 If nothing is known about the exception, this is the method which
1419 1419 should be used throughout the code for presenting user tracebacks,
1420 1420 rather than directly invoking the InteractiveTB object.
1421 1421
1422 1422 A specific showsyntaxerror() also exists, but this method can take
1423 1423 care of calling it if needed, so unless you are explicitly catching a
1424 1424 SyntaxError exception, don't try to analyze the stack manually and
1425 1425 simply call this method."""
1426 1426
1427 1427 try:
1428 1428 if exc_tuple is None:
1429 1429 etype, value, tb = sys.exc_info()
1430 1430 else:
1431 1431 etype, value, tb = exc_tuple
1432 1432
1433 1433 if etype is None:
1434 1434 if hasattr(sys, 'last_type'):
1435 1435 etype, value, tb = sys.last_type, sys.last_value, \
1436 1436 sys.last_traceback
1437 1437 else:
1438 1438 self.write_err('No traceback available to show.\n')
1439 1439 return
1440 1440
1441 1441 if etype is SyntaxError:
1442 1442 # Though this won't be called by syntax errors in the input
1443 1443 # line, there may be SyntaxError cases whith imported code.
1444 1444 self.showsyntaxerror(filename)
1445 1445 elif etype is UsageError:
1446 1446 print "UsageError:", value
1447 1447 else:
1448 1448 # WARNING: these variables are somewhat deprecated and not
1449 1449 # necessarily safe to use in a threaded environment, but tools
1450 1450 # like pdb depend on their existence, so let's set them. If we
1451 1451 # find problems in the field, we'll need to revisit their use.
1452 1452 sys.last_type = etype
1453 1453 sys.last_value = value
1454 1454 sys.last_traceback = tb
1455 1455 if etype in self.custom_exceptions:
1456 1456 # FIXME: Old custom traceback objects may just return a
1457 1457 # string, in that case we just put it into a list
1458 1458 stb = self.CustomTB(etype, value, tb, tb_offset)
1459 1459 if isinstance(ctb, basestring):
1460 1460 stb = [stb]
1461 1461 else:
1462 1462 if exception_only:
1463 1463 stb = ['An exception has occurred, use %tb to see '
1464 1464 'the full traceback.\n']
1465 1465 stb.extend(self.InteractiveTB.get_exception_only(etype,
1466 1466 value))
1467 1467 else:
1468 1468 stb = self.InteractiveTB.structured_traceback(etype,
1469 1469 value, tb, tb_offset=tb_offset)
1470 1470
1471 1471 if self.call_pdb:
1472 1472 # drop into debugger
1473 1473 self.debugger(force=True)
1474 1474
1475 1475 # Actually show the traceback
1476 1476 self._showtraceback(etype, value, stb)
1477 1477
1478 1478 except KeyboardInterrupt:
1479 1479 self.write_err("\nKeyboardInterrupt\n")
1480 1480
1481 1481 def _showtraceback(self, etype, evalue, stb):
1482 1482 """Actually show a traceback.
1483 1483
1484 1484 Subclasses may override this method to put the traceback on a different
1485 1485 place, like a side channel.
1486 1486 """
1487 1487 print >> io.Term.cout, self.InteractiveTB.stb2text(stb)
1488 1488
1489 1489 def showsyntaxerror(self, filename=None):
1490 1490 """Display the syntax error that just occurred.
1491 1491
1492 1492 This doesn't display a stack trace because there isn't one.
1493 1493
1494 1494 If a filename is given, it is stuffed in the exception instead
1495 1495 of what was there before (because Python's parser always uses
1496 1496 "<string>" when reading from a string).
1497 1497 """
1498 1498 etype, value, last_traceback = sys.exc_info()
1499 1499
1500 1500 # See note about these variables in showtraceback() above
1501 1501 sys.last_type = etype
1502 1502 sys.last_value = value
1503 1503 sys.last_traceback = last_traceback
1504 1504
1505 1505 if filename and etype is SyntaxError:
1506 1506 # Work hard to stuff the correct filename in the exception
1507 1507 try:
1508 1508 msg, (dummy_filename, lineno, offset, line) = value
1509 1509 except:
1510 1510 # Not the format we expect; leave it alone
1511 1511 pass
1512 1512 else:
1513 1513 # Stuff in the right filename
1514 1514 try:
1515 1515 # Assume SyntaxError is a class exception
1516 1516 value = SyntaxError(msg, (filename, lineno, offset, line))
1517 1517 except:
1518 1518 # If that failed, assume SyntaxError is a string
1519 1519 value = msg, (filename, lineno, offset, line)
1520 1520 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1521 1521 self._showtraceback(etype, value, stb)
1522 1522
1523 1523 #-------------------------------------------------------------------------
1524 1524 # Things related to readline
1525 1525 #-------------------------------------------------------------------------
1526 1526
1527 1527 def init_readline(self):
1528 1528 """Command history completion/saving/reloading."""
1529 1529
1530 1530 if self.readline_use:
1531 1531 import IPython.utils.rlineimpl as readline
1532 1532
1533 1533 self.rl_next_input = None
1534 1534 self.rl_do_indent = False
1535 1535
1536 1536 if not self.readline_use or not readline.have_readline:
1537 1537 self.has_readline = False
1538 1538 self.readline = None
1539 1539 # Set a number of methods that depend on readline to be no-op
1540 1540 self.set_readline_completer = no_op
1541 1541 self.set_custom_completer = no_op
1542 1542 self.set_completer_frame = no_op
1543 1543 warn('Readline services not available or not loaded.')
1544 1544 else:
1545 1545 self.has_readline = True
1546 1546 self.readline = readline
1547 1547 sys.modules['readline'] = readline
1548 1548
1549 1549 # Platform-specific configuration
1550 1550 if os.name == 'nt':
1551 1551 # FIXME - check with Frederick to see if we can harmonize
1552 1552 # naming conventions with pyreadline to avoid this
1553 1553 # platform-dependent check
1554 1554 self.readline_startup_hook = readline.set_pre_input_hook
1555 1555 else:
1556 1556 self.readline_startup_hook = readline.set_startup_hook
1557 1557
1558 1558 # Load user's initrc file (readline config)
1559 1559 # Or if libedit is used, load editrc.
1560 1560 inputrc_name = os.environ.get('INPUTRC')
1561 1561 if inputrc_name is None:
1562 1562 home_dir = get_home_dir()
1563 1563 if home_dir is not None:
1564 1564 inputrc_name = '.inputrc'
1565 1565 if readline.uses_libedit:
1566 1566 inputrc_name = '.editrc'
1567 1567 inputrc_name = os.path.join(home_dir, inputrc_name)
1568 1568 if os.path.isfile(inputrc_name):
1569 1569 try:
1570 1570 readline.read_init_file(inputrc_name)
1571 1571 except:
1572 1572 warn('Problems reading readline initialization file <%s>'
1573 1573 % inputrc_name)
1574 1574
1575 1575 # Configure readline according to user's prefs
1576 1576 # This is only done if GNU readline is being used. If libedit
1577 1577 # is being used (as on Leopard) the readline config is
1578 1578 # not run as the syntax for libedit is different.
1579 1579 if not readline.uses_libedit:
1580 1580 for rlcommand in self.readline_parse_and_bind:
1581 1581 #print "loading rl:",rlcommand # dbg
1582 1582 readline.parse_and_bind(rlcommand)
1583 1583
1584 1584 # Remove some chars from the delimiters list. If we encounter
1585 1585 # unicode chars, discard them.
1586 1586 delims = readline.get_completer_delims().encode("ascii", "ignore")
1587 1587 delims = delims.translate(None, self.readline_remove_delims)
1588 1588 delims = delims.replace(ESC_MAGIC, '')
1589 1589 readline.set_completer_delims(delims)
1590 1590 # otherwise we end up with a monster history after a while:
1591 1591 readline.set_history_length(self.history_length)
1592 1592
1593 1593 self.refill_readline_hist()
1594 1594 self.readline_no_record = ReadlineNoRecord(self)
1595 1595
1596 1596 # Configure auto-indent for all platforms
1597 1597 self.set_autoindent(self.autoindent)
1598 1598
1599 1599 def refill_readline_hist(self):
1600 1600 # Load the last 1000 lines from history
1601 1601 self.readline.clear_history()
1602 1602 stdin_encoding = sys.stdin.encoding or "utf-8"
1603 1603 for _, _, cell in self.history_manager.get_tail(1000,
1604 1604 include_latest=True):
1605 1605 if cell.strip(): # Ignore blank lines
1606 1606 for line in cell.splitlines():
1607 1607 self.readline.add_history(line.encode(stdin_encoding))
1608 1608
1609 1609 def set_next_input(self, s):
1610 1610 """ Sets the 'default' input string for the next command line.
1611 1611
1612 1612 Requires readline.
1613 1613
1614 1614 Example:
1615 1615
1616 1616 [D:\ipython]|1> _ip.set_next_input("Hello Word")
1617 1617 [D:\ipython]|2> Hello Word_ # cursor is here
1618 1618 """
1619 1619
1620 1620 self.rl_next_input = s
1621 1621
1622 1622 # Maybe move this to the terminal subclass?
1623 1623 def pre_readline(self):
1624 1624 """readline hook to be used at the start of each line.
1625 1625
1626 1626 Currently it handles auto-indent only."""
1627 1627
1628 1628 if self.rl_do_indent:
1629 1629 self.readline.insert_text(self._indent_current_str())
1630 1630 if self.rl_next_input is not None:
1631 1631 self.readline.insert_text(self.rl_next_input)
1632 1632 self.rl_next_input = None
1633 1633
1634 1634 def _indent_current_str(self):
1635 1635 """return the current level of indentation as a string"""
1636 1636 return self.input_splitter.indent_spaces * ' '
1637 1637
1638 1638 #-------------------------------------------------------------------------
1639 1639 # Things related to text completion
1640 1640 #-------------------------------------------------------------------------
1641 1641
1642 1642 def init_completer(self):
1643 1643 """Initialize the completion machinery.
1644 1644
1645 1645 This creates completion machinery that can be used by client code,
1646 1646 either interactively in-process (typically triggered by the readline
1647 1647 library), programatically (such as in test suites) or out-of-prcess
1648 1648 (typically over the network by remote frontends).
1649 1649 """
1650 1650 from IPython.core.completer import IPCompleter
1651 1651 from IPython.core.completerlib import (module_completer,
1652 1652 magic_run_completer, cd_completer)
1653 1653
1654 1654 self.Completer = IPCompleter(self,
1655 1655 self.user_ns,
1656 1656 self.user_global_ns,
1657 1657 self.readline_omit__names,
1658 1658 self.alias_manager.alias_table,
1659 1659 self.has_readline)
1660 1660
1661 1661 # Add custom completers to the basic ones built into IPCompleter
1662 1662 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1663 1663 self.strdispatchers['complete_command'] = sdisp
1664 1664 self.Completer.custom_completers = sdisp
1665 1665
1666 1666 self.set_hook('complete_command', module_completer, str_key = 'import')
1667 1667 self.set_hook('complete_command', module_completer, str_key = 'from')
1668 1668 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1669 1669 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1670 1670
1671 1671 # Only configure readline if we truly are using readline. IPython can
1672 1672 # do tab-completion over the network, in GUIs, etc, where readline
1673 1673 # itself may be absent
1674 1674 if self.has_readline:
1675 1675 self.set_readline_completer()
1676 1676
1677 1677 def complete(self, text, line=None, cursor_pos=None):
1678 1678 """Return the completed text and a list of completions.
1679 1679
1680 1680 Parameters
1681 1681 ----------
1682 1682
1683 1683 text : string
1684 1684 A string of text to be completed on. It can be given as empty and
1685 1685 instead a line/position pair are given. In this case, the
1686 1686 completer itself will split the line like readline does.
1687 1687
1688 1688 line : string, optional
1689 1689 The complete line that text is part of.
1690 1690
1691 1691 cursor_pos : int, optional
1692 1692 The position of the cursor on the input line.
1693 1693
1694 1694 Returns
1695 1695 -------
1696 1696 text : string
1697 1697 The actual text that was completed.
1698 1698
1699 1699 matches : list
1700 1700 A sorted list with all possible completions.
1701 1701
1702 1702 The optional arguments allow the completion to take more context into
1703 1703 account, and are part of the low-level completion API.
1704 1704
1705 1705 This is a wrapper around the completion mechanism, similar to what
1706 1706 readline does at the command line when the TAB key is hit. By
1707 1707 exposing it as a method, it can be used by other non-readline
1708 1708 environments (such as GUIs) for text completion.
1709 1709
1710 1710 Simple usage example:
1711 1711
1712 1712 In [1]: x = 'hello'
1713 1713
1714 1714 In [2]: _ip.complete('x.l')
1715 1715 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
1716 1716 """
1717 1717
1718 1718 # Inject names into __builtin__ so we can complete on the added names.
1719 1719 with self.builtin_trap:
1720 1720 return self.Completer.complete(text, line, cursor_pos)
1721 1721
1722 1722 def set_custom_completer(self, completer, pos=0):
1723 1723 """Adds a new custom completer function.
1724 1724
1725 1725 The position argument (defaults to 0) is the index in the completers
1726 1726 list where you want the completer to be inserted."""
1727 1727
1728 1728 newcomp = types.MethodType(completer,self.Completer)
1729 1729 self.Completer.matchers.insert(pos,newcomp)
1730 1730
1731 1731 def set_readline_completer(self):
1732 1732 """Reset readline's completer to be our own."""
1733 1733 self.readline.set_completer(self.Completer.rlcomplete)
1734 1734
1735 1735 def set_completer_frame(self, frame=None):
1736 1736 """Set the frame of the completer."""
1737 1737 if frame:
1738 1738 self.Completer.namespace = frame.f_locals
1739 1739 self.Completer.global_namespace = frame.f_globals
1740 1740 else:
1741 1741 self.Completer.namespace = self.user_ns
1742 1742 self.Completer.global_namespace = self.user_global_ns
1743 1743
1744 1744 #-------------------------------------------------------------------------
1745 1745 # Things related to magics
1746 1746 #-------------------------------------------------------------------------
1747 1747
1748 1748 def init_magics(self):
1749 1749 # FIXME: Move the color initialization to the DisplayHook, which
1750 1750 # should be split into a prompt manager and displayhook. We probably
1751 1751 # even need a centralize colors management object.
1752 1752 self.magic_colors(self.colors)
1753 1753 # History was moved to a separate module
1754 1754 from . import history
1755 1755 history.init_ipython(self)
1756 1756
1757 1757 def magic(self,arg_s):
1758 1758 """Call a magic function by name.
1759 1759
1760 1760 Input: a string containing the name of the magic function to call and
1761 1761 any additional arguments to be passed to the magic.
1762 1762
1763 1763 magic('name -opt foo bar') is equivalent to typing at the ipython
1764 1764 prompt:
1765 1765
1766 1766 In[1]: %name -opt foo bar
1767 1767
1768 1768 To call a magic without arguments, simply use magic('name').
1769 1769
1770 1770 This provides a proper Python function to call IPython's magics in any
1771 1771 valid Python code you can type at the interpreter, including loops and
1772 1772 compound statements.
1773 1773 """
1774 1774 args = arg_s.split(' ',1)
1775 1775 magic_name = args[0]
1776 1776 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
1777 1777
1778 1778 try:
1779 1779 magic_args = args[1]
1780 1780 except IndexError:
1781 1781 magic_args = ''
1782 1782 fn = getattr(self,'magic_'+magic_name,None)
1783 1783 if fn is None:
1784 1784 error("Magic function `%s` not found." % magic_name)
1785 1785 else:
1786 1786 magic_args = self.var_expand(magic_args,1)
1787 1787 # Grab local namespace if we need it:
1788 1788 if getattr(fn, "needs_local_scope", False):
1789 1789 self._magic_locals = sys._getframe(1).f_locals
1790 1790 with self.builtin_trap:
1791 1791 result = fn(magic_args)
1792 1792 # Ensure we're not keeping object references around:
1793 1793 self._magic_locals = {}
1794 1794 return result
1795 1795
1796 1796 def define_magic(self, magicname, func):
1797 1797 """Expose own function as magic function for ipython
1798 1798
1799 1799 def foo_impl(self,parameter_s=''):
1800 1800 'My very own magic!. (Use docstrings, IPython reads them).'
1801 1801 print 'Magic function. Passed parameter is between < >:'
1802 1802 print '<%s>' % parameter_s
1803 1803 print 'The self object is:',self
1804 1804
1805 1805 self.define_magic('foo',foo_impl)
1806 1806 """
1807 1807
1808 1808 import new
1809 1809 im = types.MethodType(func,self)
1810 1810 old = getattr(self, "magic_" + magicname, None)
1811 1811 setattr(self, "magic_" + magicname, im)
1812 1812 return old
1813 1813
1814 1814 #-------------------------------------------------------------------------
1815 1815 # Things related to macros
1816 1816 #-------------------------------------------------------------------------
1817 1817
1818 1818 def define_macro(self, name, themacro):
1819 1819 """Define a new macro
1820 1820
1821 1821 Parameters
1822 1822 ----------
1823 1823 name : str
1824 1824 The name of the macro.
1825 1825 themacro : str or Macro
1826 1826 The action to do upon invoking the macro. If a string, a new
1827 1827 Macro object is created by passing the string to it.
1828 1828 """
1829 1829
1830 1830 from IPython.core import macro
1831 1831
1832 1832 if isinstance(themacro, basestring):
1833 1833 themacro = macro.Macro(themacro)
1834 1834 if not isinstance(themacro, macro.Macro):
1835 1835 raise ValueError('A macro must be a string or a Macro instance.')
1836 1836 self.user_ns[name] = themacro
1837 1837
1838 1838 #-------------------------------------------------------------------------
1839 1839 # Things related to the running of system commands
1840 1840 #-------------------------------------------------------------------------
1841 1841
1842 1842 def system(self, cmd):
1843 1843 """Call the given cmd in a subprocess.
1844 1844
1845 1845 Parameters
1846 1846 ----------
1847 1847 cmd : str
1848 1848 Command to execute (can not end in '&', as bacground processes are
1849 1849 not supported.
1850 1850 """
1851 1851 # We do not support backgrounding processes because we either use
1852 1852 # pexpect or pipes to read from. Users can always just call
1853 1853 # os.system() if they really want a background process.
1854 1854 if cmd.endswith('&'):
1855 1855 raise OSError("Background processes not supported.")
1856 1856
1857 1857 return system(self.var_expand(cmd, depth=2))
1858 1858
1859 1859 def getoutput(self, cmd, split=True):
1860 1860 """Get output (possibly including stderr) from a subprocess.
1861 1861
1862 1862 Parameters
1863 1863 ----------
1864 1864 cmd : str
1865 1865 Command to execute (can not end in '&', as background processes are
1866 1866 not supported.
1867 1867 split : bool, optional
1868 1868
1869 1869 If True, split the output into an IPython SList. Otherwise, an
1870 1870 IPython LSString is returned. These are objects similar to normal
1871 1871 lists and strings, with a few convenience attributes for easier
1872 1872 manipulation of line-based output. You can use '?' on them for
1873 1873 details.
1874 1874 """
1875 1875 if cmd.endswith('&'):
1876 1876 raise OSError("Background processes not supported.")
1877 1877 out = getoutput(self.var_expand(cmd, depth=2))
1878 1878 if split:
1879 1879 out = SList(out.splitlines())
1880 1880 else:
1881 1881 out = LSString(out)
1882 1882 return out
1883 1883
1884 1884 #-------------------------------------------------------------------------
1885 1885 # Things related to aliases
1886 1886 #-------------------------------------------------------------------------
1887 1887
1888 1888 def init_alias(self):
1889 1889 self.alias_manager = AliasManager(shell=self, config=self.config)
1890 1890 self.ns_table['alias'] = self.alias_manager.alias_table,
1891 1891
1892 1892 #-------------------------------------------------------------------------
1893 1893 # Things related to extensions and plugins
1894 1894 #-------------------------------------------------------------------------
1895 1895
1896 1896 def init_extension_manager(self):
1897 1897 self.extension_manager = ExtensionManager(shell=self, config=self.config)
1898 1898
1899 1899 def init_plugin_manager(self):
1900 1900 self.plugin_manager = PluginManager(config=self.config)
1901 1901
1902 1902 #-------------------------------------------------------------------------
1903 1903 # Things related to payloads
1904 1904 #-------------------------------------------------------------------------
1905 1905
1906 1906 def init_payload(self):
1907 1907 self.payload_manager = PayloadManager(config=self.config)
1908 1908
1909 1909 #-------------------------------------------------------------------------
1910 1910 # Things related to the prefilter
1911 1911 #-------------------------------------------------------------------------
1912 1912
1913 1913 def init_prefilter(self):
1914 1914 self.prefilter_manager = PrefilterManager(shell=self, config=self.config)
1915 1915 # Ultimately this will be refactored in the new interpreter code, but
1916 1916 # for now, we should expose the main prefilter method (there's legacy
1917 1917 # code out there that may rely on this).
1918 1918 self.prefilter = self.prefilter_manager.prefilter_lines
1919 1919
1920 1920 def auto_rewrite_input(self, cmd):
1921 1921 """Print to the screen the rewritten form of the user's command.
1922 1922
1923 1923 This shows visual feedback by rewriting input lines that cause
1924 1924 automatic calling to kick in, like::
1925 1925
1926 1926 /f x
1927 1927
1928 1928 into::
1929 1929
1930 1930 ------> f(x)
1931 1931
1932 1932 after the user's input prompt. This helps the user understand that the
1933 1933 input line was transformed automatically by IPython.
1934 1934 """
1935 1935 rw = self.displayhook.prompt1.auto_rewrite() + cmd
1936 1936
1937 1937 try:
1938 1938 # plain ascii works better w/ pyreadline, on some machines, so
1939 1939 # we use it and only print uncolored rewrite if we have unicode
1940 1940 rw = str(rw)
1941 1941 print >> IPython.utils.io.Term.cout, rw
1942 1942 except UnicodeEncodeError:
1943 1943 print "------> " + cmd
1944 1944
1945 1945 #-------------------------------------------------------------------------
1946 1946 # Things related to extracting values/expressions from kernel and user_ns
1947 1947 #-------------------------------------------------------------------------
1948 1948
1949 1949 def _simple_error(self):
1950 1950 etype, value = sys.exc_info()[:2]
1951 1951 return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
1952 1952
1953 1953 def user_variables(self, names):
1954 1954 """Get a list of variable names from the user's namespace.
1955 1955
1956 1956 Parameters
1957 1957 ----------
1958 1958 names : list of strings
1959 1959 A list of names of variables to be read from the user namespace.
1960 1960
1961 1961 Returns
1962 1962 -------
1963 1963 A dict, keyed by the input names and with the repr() of each value.
1964 1964 """
1965 1965 out = {}
1966 1966 user_ns = self.user_ns
1967 1967 for varname in names:
1968 1968 try:
1969 1969 value = repr(user_ns[varname])
1970 1970 except:
1971 1971 value = self._simple_error()
1972 1972 out[varname] = value
1973 1973 return out
1974 1974
1975 1975 def user_expressions(self, expressions):
1976 1976 """Evaluate a dict of expressions in the user's namespace.
1977 1977
1978 1978 Parameters
1979 1979 ----------
1980 1980 expressions : dict
1981 1981 A dict with string keys and string values. The expression values
1982 1982 should be valid Python expressions, each of which will be evaluated
1983 1983 in the user namespace.
1984 1984
1985 1985 Returns
1986 1986 -------
1987 1987 A dict, keyed like the input expressions dict, with the repr() of each
1988 1988 value.
1989 1989 """
1990 1990 out = {}
1991 1991 user_ns = self.user_ns
1992 1992 global_ns = self.user_global_ns
1993 1993 for key, expr in expressions.iteritems():
1994 1994 try:
1995 1995 value = repr(eval(expr, global_ns, user_ns))
1996 1996 except:
1997 1997 value = self._simple_error()
1998 1998 out[key] = value
1999 1999 return out
2000 2000
2001 2001 #-------------------------------------------------------------------------
2002 2002 # Things related to the running of code
2003 2003 #-------------------------------------------------------------------------
2004 2004
2005 2005 def ex(self, cmd):
2006 2006 """Execute a normal python statement in user namespace."""
2007 2007 with self.builtin_trap:
2008 2008 exec cmd in self.user_global_ns, self.user_ns
2009 2009
2010 2010 def ev(self, expr):
2011 2011 """Evaluate python expression expr in user namespace.
2012 2012
2013 2013 Returns the result of evaluation
2014 2014 """
2015 2015 with self.builtin_trap:
2016 2016 return eval(expr, self.user_global_ns, self.user_ns)
2017 2017
2018 2018 def safe_execfile(self, fname, *where, **kw):
2019 2019 """A safe version of the builtin execfile().
2020 2020
2021 2021 This version will never throw an exception, but instead print
2022 2022 helpful error messages to the screen. This only works on pure
2023 2023 Python files with the .py extension.
2024 2024
2025 2025 Parameters
2026 2026 ----------
2027 2027 fname : string
2028 2028 The name of the file to be executed.
2029 2029 where : tuple
2030 2030 One or two namespaces, passed to execfile() as (globals,locals).
2031 2031 If only one is given, it is passed as both.
2032 2032 exit_ignore : bool (False)
2033 2033 If True, then silence SystemExit for non-zero status (it is always
2034 2034 silenced for zero status, as it is so common).
2035 2035 """
2036 2036 kw.setdefault('exit_ignore', False)
2037 2037
2038 2038 fname = os.path.abspath(os.path.expanduser(fname))
2039 2039 # Make sure we have a .py file
2040 2040 if not fname.endswith('.py'):
2041 2041 warn('File must end with .py to be run using execfile: <%s>' % fname)
2042 2042
2043 2043 # Make sure we can open the file
2044 2044 try:
2045 2045 with open(fname) as thefile:
2046 2046 pass
2047 2047 except:
2048 2048 warn('Could not open file <%s> for safe execution.' % fname)
2049 2049 return
2050 2050
2051 2051 # Find things also in current directory. This is needed to mimic the
2052 2052 # behavior of running a script from the system command line, where
2053 2053 # Python inserts the script's directory into sys.path
2054 2054 dname = os.path.dirname(fname)
2055 2055
2056 2056 if isinstance(fname, unicode):
2057 2057 # execfile uses default encoding instead of filesystem encoding
2058 2058 # so unicode filenames will fail
2059 2059 fname = fname.encode(sys.getfilesystemencoding() or sys.getdefaultencoding())
2060 2060
2061 2061 with prepended_to_syspath(dname):
2062 2062 try:
2063 2063 execfile(fname,*where)
2064 2064 except SystemExit, status:
2065 2065 # If the call was made with 0 or None exit status (sys.exit(0)
2066 2066 # or sys.exit() ), don't bother showing a traceback, as both of
2067 2067 # these are considered normal by the OS:
2068 2068 # > python -c'import sys;sys.exit(0)'; echo $?
2069 2069 # 0
2070 2070 # > python -c'import sys;sys.exit()'; echo $?
2071 2071 # 0
2072 2072 # For other exit status, we show the exception unless
2073 2073 # explicitly silenced, but only in short form.
2074 2074 if status.code not in (0, None) and not kw['exit_ignore']:
2075 2075 self.showtraceback(exception_only=True)
2076 2076 except:
2077 2077 self.showtraceback()
2078 2078
2079 2079 def safe_execfile_ipy(self, fname):
2080 2080 """Like safe_execfile, but for .ipy files with IPython syntax.
2081 2081
2082 2082 Parameters
2083 2083 ----------
2084 2084 fname : str
2085 2085 The name of the file to execute. The filename must have a
2086 2086 .ipy extension.
2087 2087 """
2088 2088 fname = os.path.abspath(os.path.expanduser(fname))
2089 2089
2090 2090 # Make sure we have a .py file
2091 2091 if not fname.endswith('.ipy'):
2092 2092 warn('File must end with .py to be run using execfile: <%s>' % fname)
2093 2093
2094 2094 # Make sure we can open the file
2095 2095 try:
2096 2096 with open(fname) as thefile:
2097 2097 pass
2098 2098 except:
2099 2099 warn('Could not open file <%s> for safe execution.' % fname)
2100 2100 return
2101 2101
2102 2102 # Find things also in current directory. This is needed to mimic the
2103 2103 # behavior of running a script from the system command line, where
2104 2104 # Python inserts the script's directory into sys.path
2105 2105 dname = os.path.dirname(fname)
2106 2106
2107 2107 with prepended_to_syspath(dname):
2108 2108 try:
2109 2109 with open(fname) as thefile:
2110 2110 # self.run_cell currently captures all exceptions
2111 2111 # raised in user code. It would be nice if there were
2112 2112 # versions of runlines, execfile that did raise, so
2113 2113 # we could catch the errors.
2114 2114 self.run_cell(thefile.read(), store_history=False)
2115 2115 except:
2116 2116 self.showtraceback()
2117 2117 warn('Unknown failure executing file: <%s>' % fname)
2118 2118
2119 2119 def run_cell(self, raw_cell, store_history=True):
2120 2120 """Run a complete IPython cell.
2121 2121
2122 2122 Parameters
2123 2123 ----------
2124 2124 raw_cell : str
2125 2125 The code (including IPython code such as %magic functions) to run.
2126 2126 store_history : bool
2127 2127 If True, the raw and translated cell will be stored in IPython's
2128 2128 history. For user code calling back into IPython's machinery, this
2129 2129 should be set to False.
2130 2130 """
2131 2131 if (not raw_cell) or raw_cell.isspace():
2132 2132 return
2133 2133
2134 2134 for line in raw_cell.splitlines():
2135 2135 self.input_splitter.push(line)
2136 2136 cell = self.input_splitter.source_reset()
2137 2137
2138 2138 with self.builtin_trap:
2139 2139 if len(cell.splitlines()) == 1:
2140 2140 cell = self.prefilter_manager.prefilter_lines(cell)
2141 2141
2142 2142 # Store raw and processed history
2143 2143 if store_history:
2144 2144 self.history_manager.store_inputs(self.execution_count,
2145 2145 cell, raw_cell)
2146 2146
2147 2147 self.logger.log(cell, raw_cell)
2148 2148
2149 2149 cell_name = self.compile.cache(cell, self.execution_count)
2150 2150
2151 2151 with self.display_trap:
2152 2152 try:
2153 2153 code_ast = ast.parse(cell, filename=cell_name)
2154 2154 except (OverflowError, SyntaxError, ValueError, TypeError,
2155 2155 MemoryError):
2156 2156 self.showsyntaxerror()
2157 2157 self.execution_count += 1
2158 2158 return None
2159 2159
2160 2160 interactivity = 'last' # Last node to be run interactive
2161 2161 if len(cell.splitlines()) == 1:
2162 2162 interactivity = 'all' # Single line; run fully interactive
2163 2163
2164 2164 self.run_ast_nodes(code_ast.body, cell_name, interactivity)
2165 2165
2166 2166 # Execute any registered post-execution functions.
2167 2167 for func, status in self._post_execute.iteritems():
2168 2168 if not status:
2169 2169 continue
2170 2170 try:
2171 2171 func()
2172 2172 except:
2173 2173 self.showtraceback()
2174 2174 # Deactivate failing function
2175 2175 self._post_execute[func] = False
2176 2176
2177 2177 if store_history:
2178 2178 # Write output to the database. Does nothing unless
2179 2179 # history output logging is enabled.
2180 2180 self.history_manager.store_output(self.execution_count)
2181 2181 # Each cell is a *single* input, regardless of how many lines it has
2182 2182 self.execution_count += 1
2183 2183
2184 2184 def run_ast_nodes(self, nodelist, cell_name, interactivity='last'):
2185 2185 """Run a sequence of AST nodes. The execution mode depends on the
2186 2186 interactivity parameter.
2187 2187
2188 2188 Parameters
2189 2189 ----------
2190 2190 nodelist : list
2191 2191 A sequence of AST nodes to run.
2192 2192 cell_name : str
2193 2193 Will be passed to the compiler as the filename of the cell. Typically
2194 2194 the value returned by ip.compile.cache(cell).
2195 2195 interactivity : str
2196 2196 'all', 'last' or 'none', specifying which nodes should be run
2197 2197 interactively (displaying output from expressions). Other values for
2198 2198 this parameter will raise a ValueError.
2199 2199 """
2200 2200 if not nodelist:
2201 2201 return
2202 2202
2203 2203 if interactivity == 'none':
2204 2204 to_run_exec, to_run_interactive = nodelist, []
2205 2205 elif interactivity == 'last':
2206 2206 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2207 2207 elif interactivity == 'all':
2208 2208 to_run_exec, to_run_interactive = [], nodelist
2209 2209 else:
2210 2210 raise ValueError("Interactivity was %r" % interactivity)
2211 2211
2212 2212 exec_count = self.execution_count
2213 2213 if to_run_exec:
2214 mod = ast.Module(to_run_exec)
2215 self.code_to_run = code = self.compile(mod, cell_name, "exec")
2216 if self.run_code(code) == 1:
2217 return
2218
2214 for i, node in enumerate(to_run_exec):
2215 mod = ast.Module([node])
2216 self.code_to_run = code = self.compile(mod, cell_name+str(i), "exec")
2217 if self.run_code(code) == 1:
2218 return 1
2219
2219 2220 if to_run_interactive:
2220 mod = ast.Interactive(to_run_interactive)
2221 self.code_to_run = code = self.compile(mod, cell_name, "single")
2222 return self.run_code(code)
2221 for i, node in enumerate(to_run_interactive):
2222 mod = ast.Interactive([node])
2223 self.code_to_run = code = self.compile(mod, cell_name, "single")
2224 if self.run_code(code) == 1:
2225 return 1
2226
2227 return 0
2223 2228
2224 2229
2225 2230 # PENDING REMOVAL: this method is slated for deletion, once our new
2226 2231 # input logic has been 100% moved to frontends and is stable.
2227 2232 def runlines(self, lines, clean=False):
2228 2233 """Run a string of one or more lines of source.
2229 2234
2230 2235 This method is capable of running a string containing multiple source
2231 2236 lines, as if they had been entered at the IPython prompt. Since it
2232 2237 exposes IPython's processing machinery, the given strings can contain
2233 2238 magic calls (%magic), special shell access (!cmd), etc.
2234 2239 """
2235 2240
2236 2241 if not isinstance(lines, (list, tuple)):
2237 2242 lines = lines.splitlines()
2238 2243
2239 2244 if clean:
2240 2245 lines = self._cleanup_ipy_script(lines)
2241 2246
2242 2247 # We must start with a clean buffer, in case this is run from an
2243 2248 # interactive IPython session (via a magic, for example).
2244 2249 self.reset_buffer()
2245 2250
2246 2251 # Since we will prefilter all lines, store the user's raw input too
2247 2252 # before we apply any transformations
2248 2253 self.buffer_raw[:] = [ l+'\n' for l in lines]
2249 2254
2250 2255 more = False
2251 2256 prefilter_lines = self.prefilter_manager.prefilter_lines
2252 2257 with nested(self.builtin_trap, self.display_trap):
2253 2258 for line in lines:
2254 2259 # skip blank lines so we don't mess up the prompt counter, but
2255 2260 # do NOT skip even a blank line if we are in a code block (more
2256 2261 # is true)
2257 2262
2258 2263 if line or more:
2259 2264 more = self.push_line(prefilter_lines(line, more))
2260 2265 # IPython's run_source returns None if there was an error
2261 2266 # compiling the code. This allows us to stop processing
2262 2267 # right away, so the user gets the error message at the
2263 2268 # right place.
2264 2269 if more is None:
2265 2270 break
2266 2271 # final newline in case the input didn't have it, so that the code
2267 2272 # actually does get executed
2268 2273 if more:
2269 2274 self.push_line('\n')
2270 2275
2271 2276 def run_source(self, source, filename=None, symbol='single'):
2272 2277 """Compile and run some source in the interpreter.
2273 2278
2274 2279 Arguments are as for compile_command().
2275 2280
2276 2281 One several things can happen:
2277 2282
2278 2283 1) The input is incorrect; compile_command() raised an
2279 2284 exception (SyntaxError or OverflowError). A syntax traceback
2280 2285 will be printed by calling the showsyntaxerror() method.
2281 2286
2282 2287 2) The input is incomplete, and more input is required;
2283 2288 compile_command() returned None. Nothing happens.
2284 2289
2285 2290 3) The input is complete; compile_command() returned a code
2286 2291 object. The code is executed by calling self.run_code() (which
2287 2292 also handles run-time exceptions, except for SystemExit).
2288 2293
2289 2294 The return value is:
2290 2295
2291 2296 - True in case 2
2292 2297
2293 2298 - False in the other cases, unless an exception is raised, where
2294 2299 None is returned instead. This can be used by external callers to
2295 2300 know whether to continue feeding input or not.
2296 2301
2297 2302 The return value can be used to decide whether to use sys.ps1 or
2298 2303 sys.ps2 to prompt the next line."""
2299 2304
2300 2305 # We need to ensure that the source is unicode from here on.
2301 2306 if type(source)==str:
2302 2307 usource = source.decode(self.stdin_encoding)
2303 2308 else:
2304 2309 usource = source
2305 2310
2306 2311 try:
2307 2312 code_name = self.compile.cache(usource, self.execution_count)
2308 2313 code = self.compile(usource, code_name, symbol)
2309 2314 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
2310 2315 # Case 1
2311 2316 self.showsyntaxerror(filename)
2312 2317 return None
2313 2318
2314 2319 if code is None:
2315 2320 # Case 2
2316 2321 return True
2317 2322
2318 2323 # Case 3
2319 2324 # We store the code object so that threaded shells and
2320 2325 # custom exception handlers can access all this info if needed.
2321 2326 # The source corresponding to this can be obtained from the
2322 2327 # buffer attribute as '\n'.join(self.buffer).
2323 2328 self.code_to_run = code
2324 2329 # now actually execute the code object
2325 2330 if self.run_code(code) == 0:
2326 2331 return False
2327 2332 else:
2328 2333 return None
2329 2334
2330 2335 # For backwards compatibility
2331 2336 runsource = run_source
2332 2337
2333 2338 def run_code(self, code_obj):
2334 2339 """Execute a code object.
2335 2340
2336 2341 When an exception occurs, self.showtraceback() is called to display a
2337 2342 traceback.
2338 2343
2339 Return value: a flag indicating whether the code to be run completed
2340 successfully:
2344 Parameters
2345 ----------
2346 code_obj : code object
2347 A compiled code object, to be executed
2348 post_execute : bool [default: True]
2349 whether to call post_execute hooks after this particular execution.
2341 2350
2342 - 0: successful execution.
2343 - 1: an error occurred.
2351 Returns
2352 -------
2353 0 : successful execution.
2354 1 : an error occurred.
2344 2355 """
2345 2356
2346 2357 # Set our own excepthook in case the user code tries to call it
2347 2358 # directly, so that the IPython crash handler doesn't get triggered
2348 2359 old_excepthook,sys.excepthook = sys.excepthook, self.excepthook
2349 2360
2350 2361 # we save the original sys.excepthook in the instance, in case config
2351 2362 # code (such as magics) needs access to it.
2352 2363 self.sys_excepthook = old_excepthook
2353 2364 outflag = 1 # happens in more places, so it's easier as default
2354 2365 try:
2355 2366 try:
2356 2367 self.hooks.pre_run_code_hook()
2357 2368 #rprint('Running code', repr(code_obj)) # dbg
2358 2369 exec code_obj in self.user_global_ns, self.user_ns
2359 2370 finally:
2360 2371 # Reset our crash handler in place
2361 2372 sys.excepthook = old_excepthook
2362 2373 except SystemExit:
2363 2374 self.reset_buffer()
2364 2375 self.showtraceback(exception_only=True)
2365 2376 warn("To exit: use any of 'exit', 'quit', %Exit or Ctrl-D.", level=1)
2366 2377 except self.custom_exceptions:
2367 2378 etype,value,tb = sys.exc_info()
2368 2379 self.CustomTB(etype,value,tb)
2369 2380 except:
2370 2381 self.showtraceback()
2371 2382 else:
2372 2383 outflag = 0
2373 2384 if softspace(sys.stdout, 0):
2374 2385 print
2375 2386
2376 2387 # Flush out code object which has been run (and source)
2377 2388 self.code_to_run = None
2378 2389 return outflag
2379 2390
2380 2391 # For backwards compatibility
2381 2392 runcode = run_code
2382 2393
2383 2394 # PENDING REMOVAL: this method is slated for deletion, once our new
2384 2395 # input logic has been 100% moved to frontends and is stable.
2385 2396 def push_line(self, line):
2386 2397 """Push a line to the interpreter.
2387 2398
2388 2399 The line should not have a trailing newline; it may have
2389 2400 internal newlines. The line is appended to a buffer and the
2390 2401 interpreter's run_source() method is called with the
2391 2402 concatenated contents of the buffer as source. If this
2392 2403 indicates that the command was executed or invalid, the buffer
2393 2404 is reset; otherwise, the command is incomplete, and the buffer
2394 2405 is left as it was after the line was appended. The return
2395 2406 value is 1 if more input is required, 0 if the line was dealt
2396 2407 with in some way (this is the same as run_source()).
2397 2408 """
2398 2409
2399 2410 # autoindent management should be done here, and not in the
2400 2411 # interactive loop, since that one is only seen by keyboard input. We
2401 2412 # need this done correctly even for code run via runlines (which uses
2402 2413 # push).
2403 2414
2404 2415 #print 'push line: <%s>' % line # dbg
2405 2416 self.buffer.append(line)
2406 2417 full_source = '\n'.join(self.buffer)
2407 2418 more = self.run_source(full_source, self.filename)
2408 2419 if not more:
2409 2420 self.history_manager.store_inputs(self.execution_count,
2410 2421 '\n'.join(self.buffer_raw), full_source)
2411 2422 self.reset_buffer()
2412 2423 self.execution_count += 1
2413 2424 return more
2414 2425
2415 2426 def reset_buffer(self):
2416 2427 """Reset the input buffer."""
2417 2428 self.buffer[:] = []
2418 2429 self.buffer_raw[:] = []
2419 2430 self.input_splitter.reset()
2420 2431
2421 2432 # For backwards compatibility
2422 2433 resetbuffer = reset_buffer
2423 2434
2424 2435 def _is_secondary_block_start(self, s):
2425 2436 if not s.endswith(':'):
2426 2437 return False
2427 2438 if (s.startswith('elif') or
2428 2439 s.startswith('else') or
2429 2440 s.startswith('except') or
2430 2441 s.startswith('finally')):
2431 2442 return True
2432 2443
2433 2444 def _cleanup_ipy_script(self, script):
2434 2445 """Make a script safe for self.runlines()
2435 2446
2436 2447 Currently, IPython is lines based, with blocks being detected by
2437 2448 empty lines. This is a problem for block based scripts that may
2438 2449 not have empty lines after blocks. This script adds those empty
2439 2450 lines to make scripts safe for running in the current line based
2440 2451 IPython.
2441 2452 """
2442 2453 res = []
2443 2454 lines = script.splitlines()
2444 2455 level = 0
2445 2456
2446 2457 for l in lines:
2447 2458 lstripped = l.lstrip()
2448 2459 stripped = l.strip()
2449 2460 if not stripped:
2450 2461 continue
2451 2462 newlevel = len(l) - len(lstripped)
2452 2463 if level > 0 and newlevel == 0 and \
2453 2464 not self._is_secondary_block_start(stripped):
2454 2465 # add empty line
2455 2466 res.append('')
2456 2467 res.append(l)
2457 2468 level = newlevel
2458 2469
2459 2470 return '\n'.join(res) + '\n'
2460 2471
2461 2472 #-------------------------------------------------------------------------
2462 2473 # Things related to GUI support and pylab
2463 2474 #-------------------------------------------------------------------------
2464 2475
2465 2476 def enable_pylab(self, gui=None):
2466 2477 raise NotImplementedError('Implement enable_pylab in a subclass')
2467 2478
2468 2479 #-------------------------------------------------------------------------
2469 2480 # Utilities
2470 2481 #-------------------------------------------------------------------------
2471 2482
2472 2483 def var_expand(self,cmd,depth=0):
2473 2484 """Expand python variables in a string.
2474 2485
2475 2486 The depth argument indicates how many frames above the caller should
2476 2487 be walked to look for the local namespace where to expand variables.
2477 2488
2478 2489 The global namespace for expansion is always the user's interactive
2479 2490 namespace.
2480 2491 """
2481 2492 res = ItplNS(cmd, self.user_ns, # globals
2482 2493 # Skip our own frame in searching for locals:
2483 2494 sys._getframe(depth+1).f_locals # locals
2484 2495 )
2485 2496 return str(res).decode(res.codec)
2486 2497
2487 2498 def mktempfile(self, data=None, prefix='ipython_edit_'):
2488 2499 """Make a new tempfile and return its filename.
2489 2500
2490 2501 This makes a call to tempfile.mktemp, but it registers the created
2491 2502 filename internally so ipython cleans it up at exit time.
2492 2503
2493 2504 Optional inputs:
2494 2505
2495 2506 - data(None): if data is given, it gets written out to the temp file
2496 2507 immediately, and the file is closed again."""
2497 2508
2498 2509 filename = tempfile.mktemp('.py', prefix)
2499 2510 self.tempfiles.append(filename)
2500 2511
2501 2512 if data:
2502 2513 tmp_file = open(filename,'w')
2503 2514 tmp_file.write(data)
2504 2515 tmp_file.close()
2505 2516 return filename
2506 2517
2507 2518 # TODO: This should be removed when Term is refactored.
2508 2519 def write(self,data):
2509 2520 """Write a string to the default output"""
2510 2521 io.Term.cout.write(data)
2511 2522
2512 2523 # TODO: This should be removed when Term is refactored.
2513 2524 def write_err(self,data):
2514 2525 """Write a string to the default error output"""
2515 2526 io.Term.cerr.write(data)
2516 2527
2517 2528 def ask_yes_no(self,prompt,default=True):
2518 2529 if self.quiet:
2519 2530 return True
2520 2531 return ask_yes_no(prompt,default)
2521 2532
2522 2533 def show_usage(self):
2523 2534 """Show a usage message"""
2524 2535 page.page(IPython.core.usage.interactive_usage)
2525 2536
2526 2537 def find_user_code(self, target, raw=True):
2527 2538 """Get a code string from history, file, or a string or macro.
2528 2539
2529 2540 This is mainly used by magic functions.
2530 2541
2531 2542 Parameters
2532 2543 ----------
2533 2544 target : str
2534 2545 A string specifying code to retrieve. This will be tried respectively
2535 2546 as: ranges of input history (see %history for syntax), a filename, or
2536 2547 an expression evaluating to a string or Macro in the user namespace.
2537 2548 raw : bool
2538 2549 If true (default), retrieve raw history. Has no effect on the other
2539 2550 retrieval mechanisms.
2540 2551
2541 2552 Returns
2542 2553 -------
2543 2554 A string of code.
2544 2555
2545 2556 ValueError is raised if nothing is found, and TypeError if it evaluates
2546 2557 to an object of another type. In each case, .args[0] is a printable
2547 2558 message.
2548 2559 """
2549 2560 code = self.extract_input_lines(target, raw=raw) # Grab history
2550 2561 if code:
2551 2562 return code
2552 2563 if os.path.isfile(target): # Read file
2553 2564 return open(target, "r").read()
2554 2565
2555 2566 try: # User namespace
2556 2567 codeobj = eval(target, self.user_ns)
2557 2568 except Exception:
2558 2569 raise ValueError(("'%s' was not found in history, as a file, nor in"
2559 2570 " the user namespace.") % target)
2560 2571 if isinstance(codeobj, basestring):
2561 2572 return codeobj
2562 2573 elif isinstance(codeobj, Macro):
2563 2574 return codeobj.value
2564 2575
2565 2576 raise TypeError("%s is neither a string nor a macro." % target,
2566 2577 codeobj)
2567 2578
2568 2579 #-------------------------------------------------------------------------
2569 2580 # Things related to IPython exiting
2570 2581 #-------------------------------------------------------------------------
2571 2582 def atexit_operations(self):
2572 2583 """This will be executed at the time of exit.
2573 2584
2574 2585 Cleanup operations and saving of persistent data that is done
2575 2586 unconditionally by IPython should be performed here.
2576 2587
2577 2588 For things that may depend on startup flags or platform specifics (such
2578 2589 as having readline or not), register a separate atexit function in the
2579 2590 code that has the appropriate information, rather than trying to
2580 2591 clutter
2581 2592 """
2582 2593 # Cleanup all tempfiles left around
2583 2594 for tfile in self.tempfiles:
2584 2595 try:
2585 2596 os.unlink(tfile)
2586 2597 except OSError:
2587 2598 pass
2588 2599
2589 2600 # Close the history session (this stores the end time and line count)
2590 2601 self.history_manager.end_session()
2591 2602
2592 2603 # Clear all user namespaces to release all references cleanly.
2593 2604 self.reset(new_session=False)
2594 2605
2595 2606 # Run user hooks
2596 2607 self.hooks.shutdown_hook()
2597 2608
2598 2609 def cleanup(self):
2599 2610 self.restore_sys_module_state()
2600 2611
2601 2612
2602 2613 class InteractiveShellABC(object):
2603 2614 """An abstract base class for InteractiveShell."""
2604 2615 __metaclass__ = abc.ABCMeta
2605 2616
2606 2617 InteractiveShellABC.register(InteractiveShell)
@@ -1,238 +1,276 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3
4 4 """Magic command interface for interactive parallel work."""
5 5
6 6 #-----------------------------------------------------------------------------
7 7 # Copyright (C) 2008-2009 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 import ast
18 import new
19 18 import re
20 19
21 20 from IPython.core.plugin import Plugin
22 21 from IPython.utils.traitlets import Bool, Any, Instance
23 from IPython.utils.autoattr import auto_attr
24 22 from IPython.testing import decorators as testdec
25 23
26 24 #-----------------------------------------------------------------------------
27 25 # Definitions of magic functions for use with IPython
28 26 #-----------------------------------------------------------------------------
29 27
30 28
31 29 NO_ACTIVE_VIEW = """
32 30 Use activate() on a DirectView object to activate it for magics.
33 31 """
34 32
35 33
36 34 class ParalleMagic(Plugin):
37 35 """A component to manage the %result, %px and %autopx magics."""
38 36
39 37 active_view = Any()
40 38 verbose = Bool(False, config=True)
41 39 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
42 40
43 41 def __init__(self, shell=None, config=None):
44 42 super(ParalleMagic, self).__init__(shell=shell, config=config)
45 43 self._define_magics()
46 44 # A flag showing if autopx is activated or not
47 45 self.autopx = False
48 46
49 47 def _define_magics(self):
50 48 """Define the magic functions."""
51 49 self.shell.define_magic('result', self.magic_result)
52 50 self.shell.define_magic('px', self.magic_px)
53 51 self.shell.define_magic('autopx', self.magic_autopx)
54 52
55 53 @testdec.skip_doctest
56 54 def magic_result(self, ipself, parameter_s=''):
57 55 """Print the result of command i on all engines..
58 56
59 57 To use this a :class:`DirectView` instance must be created
60 58 and then activated by calling its :meth:`activate` method.
61 59
62 60 Then you can do the following::
63 61
64 62 In [23]: %result
65 63 Out[23]:
66 64 <Results List>
67 65 [0] In [6]: a = 10
68 66 [1] In [6]: a = 10
69 67
70 68 In [22]: %result 6
71 69 Out[22]:
72 70 <Results List>
73 71 [0] In [6]: a = 10
74 72 [1] In [6]: a = 10
75 73 """
76 74 if self.active_view is None:
77 75 print NO_ACTIVE_VIEW
78 76 return
79 77
80 78 try:
81 79 index = int(parameter_s)
82 80 except:
83 81 index = None
84 82 result = self.active_view.get_result(index)
85 83 return result
86 84
87 85 @testdec.skip_doctest
88 86 def magic_px(self, ipself, parameter_s=''):
89 87 """Executes the given python command in parallel.
90 88
91 89 To use this a :class:`DirectView` instance must be created
92 90 and then activated by calling its :meth:`activate` method.
93 91
94 92 Then you can do the following::
95 93
96 94 In [24]: %px a = 5
97 95 Parallel execution on engines: all
98 96 Out[24]:
99 97 <Results List>
100 98 [0] In [7]: a = 5
101 99 [1] In [7]: a = 5
102 100 """
103 101
104 102 if self.active_view is None:
105 103 print NO_ACTIVE_VIEW
106 104 return
107 105 print "Parallel execution on engines: %s" % self.active_view.targets
108 106 result = self.active_view.execute(parameter_s)
109 107 return result
110 108
111 109 @testdec.skip_doctest
112 110 def magic_autopx(self, ipself, parameter_s=''):
113 111 """Toggles auto parallel mode.
114 112
115 113 To use this a :class:`DirectView` instance must be created
116 114 and then activated by calling its :meth:`activate` method. Once this
117 115 is called, all commands typed at the command line are send to
118 116 the engines to be executed in parallel. To control which engine
119 117 are used, set the ``targets`` attributed of the multiengine client
120 118 before entering ``%autopx`` mode.
121 119
122 120 Then you can do the following::
123 121
124 122 In [25]: %autopx
125 123 %autopx to enabled
126 124
127 125 In [26]: a = 10
128 126 <Results List>
129 127 [0] In [8]: a = 10
130 128 [1] In [8]: a = 10
131 129
132 130
133 131 In [27]: %autopx
134 132 %autopx disabled
135 133 """
136 134 if self.autopx:
137 135 self._disable_autopx()
138 136 else:
139 137 self._enable_autopx()
140 138
141 139 def _enable_autopx(self):
142 140 """Enable %autopx mode by saving the original run_cell and installing
143 141 pxrun_cell.
144 142 """
145 143 if self.active_view is None:
146 144 print NO_ACTIVE_VIEW
147 145 return
148 146
147 # override run_cell and run_code
149 148 self._original_run_cell = self.shell.run_cell
150 self.shell.run_cell = new.instancemethod(
151 self.pxrun_cell, self.shell, self.shell.__class__
152 )
149 self.shell.run_cell = self.pxrun_cell
150 self._original_run_code = self.shell.run_code
151 self.shell.run_code = self.pxrun_code
152
153 153 self.autopx = True
154 154 print "%autopx enabled"
155 155
156 156 def _disable_autopx(self):
157 157 """Disable %autopx by restoring the original InteractiveShell.run_cell.
158 158 """
159 159 if self.autopx:
160 160 self.shell.run_cell = self._original_run_cell
161 self.shell.run_code = self._original_run_code
161 162 self.autopx = False
162 163 print "%autopx disabled"
163 164
164 def pxrun_cell(self, ipself, cell, store_history=True):
165 def _maybe_display_output(self, result):
166 """Maybe display the output of a parallel result.
167
168 If self.active_view.block is True, wait for the result
169 and display the result. Otherwise, this is a noop.
170 """
171 if self.active_view.block:
172 try:
173 result.get()
174 except:
175 self.shell.showtraceback()
176 return True
177 else:
178 targets = self.active_view.targets
179 if isinstance(targets, int):
180 targets = [targets]
181 if targets == 'all':
182 targets = self.active_view.client.ids
183 stdout = [s.rstrip() for s in result.stdout]
184 if any(stdout):
185 for i,eid in enumerate(targets):
186 print '[stdout:%i]'%eid, stdout[i]
187 return False
188
189
190 def pxrun_cell(self, cell, store_history=True):
165 191 """drop-in replacement for InteractiveShell.run_cell.
166 192
167 193 This executes code remotely, instead of in the local namespace.
194
195 See InteractiveShell.run_cell for details.
168 196 """
197 ipself = self.shell
169 198 raw_cell = cell
170 199 with ipself.builtin_trap:
171 200 cell = ipself.prefilter_manager.prefilter_lines(cell)
172 201
173 202 # Store raw and processed history
174 203 if store_history:
175 204 ipself.history_manager.store_inputs(ipself.execution_count,
176 205 cell, raw_cell)
177 206
178 207 # ipself.logger.log(cell, raw_cell)
179 208
180 209 cell_name = ipself.compile.cache(cell, ipself.execution_count)
181 210
182 211 try:
183 212 code_ast = ast.parse(cell, filename=cell_name)
184 213 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
185 214 # Case 1
186 215 ipself.showsyntaxerror()
187 216 ipself.execution_count += 1
188 217 return None
189 218 except NameError:
219 # ignore name errors, because we don't know the remote keys
190 220 pass
191 221
192 222 if store_history:
193 223 # Write output to the database. Does nothing unless
194 224 # history output logging is enabled.
195 225 ipself.history_manager.store_output(ipself.execution_count)
196 226 # Each cell is a *single* input, regardless of how many lines it has
197 227 ipself.execution_count += 1
198 print cell
199 228
200 229 if re.search(r'get_ipython\(\)\.magic\(u?"%?autopx', cell):
201 230 self._disable_autopx()
202 231 return False
203 232 else:
204 233 try:
205 234 result = self.active_view.execute(cell, block=False)
206 235 except:
207 236 ipself.showtraceback()
208 237 return False
209
210 if self.active_view.block:
211 try:
212 result.get()
213 except:
214 ipself.showtraceback()
215 else:
216 targets = self.active_view.targets
217 if isinstance(targets, int):
218 targets = [targets]
219 if targets == 'all':
220 targets = self.active_view.client.ids
221 stdout = [s.rstrip() for s in result.stdout]
222 if any(stdout):
223 for i,eid in enumerate(targets):
224 print '[stdout:%i]'%eid, stdout[i]
238 else:
239 return self._maybe_display_output(result)
240
241 def pxrun_code(self, code_obj, post_execute=True):
242 """drop-in replacement for InteractiveShell.run_code.
243
244 This executes code remotely, instead of in the local namespace.
245
246 See InteractiveShell.run_code for details.
247 """
248 ipself = self.shell
249 # check code object for the autopx magic
250 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names and \
251 any( [ isinstance(c, basestring) and 'autopx' in c for c in code_obj.co_consts ]):
252 self._disable_autopx()
225 253 return False
254 else:
255 try:
256 result = self.active_view.execute(code_obj, block=False)
257 except:
258 ipself.showtraceback()
259 return False
260 else:
261 return self._maybe_display_output(result)
262
263
226 264
227 265
228 266 _loaded = False
229 267
230 268
231 269 def load_ipython_extension(ip):
232 270 """Load the extension in IPython."""
233 271 global _loaded
234 272 if not _loaded:
235 273 plugin = ParalleMagic(shell=ip, config=ip.config)
236 274 ip.plugin_manager.register_plugin('parallelmagic', plugin)
237 275 _loaded = True
238 276
@@ -1,1033 +1,1033 b''
1 1 """Views of remote engines."""
2 2 #-----------------------------------------------------------------------------
3 3 # Copyright (C) 2010 The IPython Development Team
4 4 #
5 5 # Distributed under the terms of the BSD License. The full license is in
6 6 # the file COPYING, distributed as part of this software.
7 7 #-----------------------------------------------------------------------------
8 8
9 9 #-----------------------------------------------------------------------------
10 10 # Imports
11 11 #-----------------------------------------------------------------------------
12 12
13 13 import imp
14 14 import sys
15 15 import warnings
16 16 from contextlib import contextmanager
17 17 from types import ModuleType
18 18
19 19 import zmq
20 20
21 21 from IPython.testing import decorators as testdec
22 22 from IPython.utils.traitlets import HasTraits, Any, Bool, List, Dict, Set, Int, Instance, CFloat
23 23
24 24 from IPython.external.decorator import decorator
25 25
26 26 from IPython.parallel import util
27 27 from IPython.parallel.controller.dependency import Dependency, dependent
28 28
29 29 from . import map as Map
30 30 from .asyncresult import AsyncResult, AsyncMapResult
31 31 from .remotefunction import ParallelFunction, parallel, remote
32 32
33 33 #-----------------------------------------------------------------------------
34 34 # Decorators
35 35 #-----------------------------------------------------------------------------
36 36
37 37 @decorator
38 38 def save_ids(f, self, *args, **kwargs):
39 39 """Keep our history and outstanding attributes up to date after a method call."""
40 40 n_previous = len(self.client.history)
41 41 try:
42 42 ret = f(self, *args, **kwargs)
43 43 finally:
44 44 nmsgs = len(self.client.history) - n_previous
45 45 msg_ids = self.client.history[-nmsgs:]
46 46 self.history.extend(msg_ids)
47 47 map(self.outstanding.add, msg_ids)
48 48 return ret
49 49
50 50 @decorator
51 51 def sync_results(f, self, *args, **kwargs):
52 52 """sync relevant results from self.client to our results attribute."""
53 53 ret = f(self, *args, **kwargs)
54 54 delta = self.outstanding.difference(self.client.outstanding)
55 55 completed = self.outstanding.intersection(delta)
56 56 self.outstanding = self.outstanding.difference(completed)
57 57 for msg_id in completed:
58 58 self.results[msg_id] = self.client.results[msg_id]
59 59 return ret
60 60
61 61 @decorator
62 62 def spin_after(f, self, *args, **kwargs):
63 63 """call spin after the method."""
64 64 ret = f(self, *args, **kwargs)
65 65 self.spin()
66 66 return ret
67 67
68 68 #-----------------------------------------------------------------------------
69 69 # Classes
70 70 #-----------------------------------------------------------------------------
71 71
72 72 @testdec.skip_doctest
73 73 class View(HasTraits):
74 74 """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
75 75
76 76 Don't use this class, use subclasses.
77 77
78 78 Methods
79 79 -------
80 80
81 81 spin
82 82 flushes incoming results and registration state changes
83 83 control methods spin, and requesting `ids` also ensures up to date
84 84
85 85 wait
86 86 wait on one or more msg_ids
87 87
88 88 execution methods
89 89 apply
90 90 legacy: execute, run
91 91
92 92 data movement
93 93 push, pull, scatter, gather
94 94
95 95 query methods
96 96 get_result, queue_status, purge_results, result_status
97 97
98 98 control methods
99 99 abort, shutdown
100 100
101 101 """
102 102 # flags
103 103 block=Bool(False)
104 104 track=Bool(True)
105 105 targets = Any()
106 106
107 107 history=List()
108 108 outstanding = Set()
109 109 results = Dict()
110 110 client = Instance('IPython.parallel.Client')
111 111
112 112 _socket = Instance('zmq.Socket')
113 113 _flag_names = List(['targets', 'block', 'track'])
114 114 _targets = Any()
115 115 _idents = Any()
116 116
117 117 def __init__(self, client=None, socket=None, **flags):
118 118 super(View, self).__init__(client=client, _socket=socket)
119 119 self.block = client.block
120 120
121 121 self.set_flags(**flags)
122 122
123 123 assert not self.__class__ is View, "Don't use base View objects, use subclasses"
124 124
125 125
126 126 def __repr__(self):
127 127 strtargets = str(self.targets)
128 128 if len(strtargets) > 16:
129 129 strtargets = strtargets[:12]+'...]'
130 130 return "<%s %s>"%(self.__class__.__name__, strtargets)
131 131
132 132 def set_flags(self, **kwargs):
133 133 """set my attribute flags by keyword.
134 134
135 135 Views determine behavior with a few attributes (`block`, `track`, etc.).
136 136 These attributes can be set all at once by name with this method.
137 137
138 138 Parameters
139 139 ----------
140 140
141 141 block : bool
142 142 whether to wait for results
143 143 track : bool
144 144 whether to create a MessageTracker to allow the user to
145 145 safely edit after arrays and buffers during non-copying
146 146 sends.
147 147 """
148 148 for name, value in kwargs.iteritems():
149 149 if name not in self._flag_names:
150 150 raise KeyError("Invalid name: %r"%name)
151 151 else:
152 152 setattr(self, name, value)
153 153
154 154 @contextmanager
155 155 def temp_flags(self, **kwargs):
156 156 """temporarily set flags, for use in `with` statements.
157 157
158 158 See set_flags for permanent setting of flags
159 159
160 160 Examples
161 161 --------
162 162
163 163 >>> view.track=False
164 164 ...
165 165 >>> with view.temp_flags(track=True):
166 166 ... ar = view.apply(dostuff, my_big_array)
167 167 ... ar.tracker.wait() # wait for send to finish
168 168 >>> view.track
169 169 False
170 170
171 171 """
172 172 # preflight: save flags, and set temporaries
173 173 saved_flags = {}
174 174 for f in self._flag_names:
175 175 saved_flags[f] = getattr(self, f)
176 176 self.set_flags(**kwargs)
177 177 # yield to the with-statement block
178 178 try:
179 179 yield
180 180 finally:
181 181 # postflight: restore saved flags
182 182 self.set_flags(**saved_flags)
183 183
184 184
185 185 #----------------------------------------------------------------
186 186 # apply
187 187 #----------------------------------------------------------------
188 188
189 189 @sync_results
190 190 @save_ids
191 191 def _really_apply(self, f, args, kwargs, block=None, **options):
192 192 """wrapper for client.send_apply_message"""
193 193 raise NotImplementedError("Implement in subclasses")
194 194
195 195 def apply(self, f, *args, **kwargs):
196 196 """calls f(*args, **kwargs) on remote engines, returning the result.
197 197
198 198 This method sets all apply flags via this View's attributes.
199 199
200 200 if self.block is False:
201 201 returns AsyncResult
202 202 else:
203 203 returns actual result of f(*args, **kwargs)
204 204 """
205 205 return self._really_apply(f, args, kwargs)
206 206
207 207 def apply_async(self, f, *args, **kwargs):
208 208 """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
209 209
210 210 returns AsyncResult
211 211 """
212 212 return self._really_apply(f, args, kwargs, block=False)
213 213
214 214 @spin_after
215 215 def apply_sync(self, f, *args, **kwargs):
216 216 """calls f(*args, **kwargs) on remote engines in a blocking manner,
217 217 returning the result.
218 218
219 219 returns: actual result of f(*args, **kwargs)
220 220 """
221 221 return self._really_apply(f, args, kwargs, block=True)
222 222
223 223 #----------------------------------------------------------------
224 224 # wrappers for client and control methods
225 225 #----------------------------------------------------------------
226 226 @sync_results
227 227 def spin(self):
228 228 """spin the client, and sync"""
229 229 self.client.spin()
230 230
231 231 @sync_results
232 232 def wait(self, jobs=None, timeout=-1):
233 233 """waits on one or more `jobs`, for up to `timeout` seconds.
234 234
235 235 Parameters
236 236 ----------
237 237
238 238 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
239 239 ints are indices to self.history
240 240 strs are msg_ids
241 241 default: wait on all outstanding messages
242 242 timeout : float
243 243 a time in seconds, after which to give up.
244 244 default is -1, which means no timeout
245 245
246 246 Returns
247 247 -------
248 248
249 249 True : when all msg_ids are done
250 250 False : timeout reached, some msg_ids still outstanding
251 251 """
252 252 if jobs is None:
253 253 jobs = self.history
254 254 return self.client.wait(jobs, timeout)
255 255
256 256 def abort(self, jobs=None, targets=None, block=None):
257 257 """Abort jobs on my engines.
258 258
259 259 Parameters
260 260 ----------
261 261
262 262 jobs : None, str, list of strs, optional
263 263 if None: abort all jobs.
264 264 else: abort specific msg_id(s).
265 265 """
266 266 block = block if block is not None else self.block
267 267 targets = targets if targets is not None else self.targets
268 268 return self.client.abort(jobs=jobs, targets=targets, block=block)
269 269
270 270 def queue_status(self, targets=None, verbose=False):
271 271 """Fetch the Queue status of my engines"""
272 272 targets = targets if targets is not None else self.targets
273 273 return self.client.queue_status(targets=targets, verbose=verbose)
274 274
275 275 def purge_results(self, jobs=[], targets=[]):
276 276 """Instruct the controller to forget specific results."""
277 277 if targets is None or targets == 'all':
278 278 targets = self.targets
279 279 return self.client.purge_results(jobs=jobs, targets=targets)
280 280
281 281 def shutdown(self, targets=None, restart=False, hub=False, block=None):
282 282 """Terminates one or more engine processes, optionally including the hub.
283 283 """
284 284 block = self.block if block is None else block
285 285 if targets is None or targets == 'all':
286 286 targets = self.targets
287 287 return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
288 288
289 289 @spin_after
290 290 def get_result(self, indices_or_msg_ids=None):
291 291 """return one or more results, specified by history index or msg_id.
292 292
293 293 See client.get_result for details.
294 294
295 295 """
296 296
297 297 if indices_or_msg_ids is None:
298 298 indices_or_msg_ids = -1
299 299 if isinstance(indices_or_msg_ids, int):
300 300 indices_or_msg_ids = self.history[indices_or_msg_ids]
301 301 elif isinstance(indices_or_msg_ids, (list,tuple,set)):
302 302 indices_or_msg_ids = list(indices_or_msg_ids)
303 303 for i,index in enumerate(indices_or_msg_ids):
304 304 if isinstance(index, int):
305 305 indices_or_msg_ids[i] = self.history[index]
306 306 return self.client.get_result(indices_or_msg_ids)
307 307
308 308 #-------------------------------------------------------------------
309 309 # Map
310 310 #-------------------------------------------------------------------
311 311
312 312 def map(self, f, *sequences, **kwargs):
313 313 """override in subclasses"""
314 314 raise NotImplementedError
315 315
316 316 def map_async(self, f, *sequences, **kwargs):
317 317 """Parallel version of builtin `map`, using this view's engines.
318 318
319 319 This is equivalent to map(...block=False)
320 320
321 321 See `self.map` for details.
322 322 """
323 323 if 'block' in kwargs:
324 324 raise TypeError("map_async doesn't take a `block` keyword argument.")
325 325 kwargs['block'] = False
326 326 return self.map(f,*sequences,**kwargs)
327 327
328 328 def map_sync(self, f, *sequences, **kwargs):
329 329 """Parallel version of builtin `map`, using this view's engines.
330 330
331 331 This is equivalent to map(...block=True)
332 332
333 333 See `self.map` for details.
334 334 """
335 335 if 'block' in kwargs:
336 336 raise TypeError("map_sync doesn't take a `block` keyword argument.")
337 337 kwargs['block'] = True
338 338 return self.map(f,*sequences,**kwargs)
339 339
340 340 def imap(self, f, *sequences, **kwargs):
341 341 """Parallel version of `itertools.imap`.
342 342
343 343 See `self.map` for details.
344 344
345 345 """
346 346
347 347 return iter(self.map_async(f,*sequences, **kwargs))
348 348
349 349 #-------------------------------------------------------------------
350 350 # Decorators
351 351 #-------------------------------------------------------------------
352 352
353 353 def remote(self, block=True, **flags):
354 354 """Decorator for making a RemoteFunction"""
355 355 block = self.block if block is None else block
356 356 return remote(self, block=block, **flags)
357 357
358 358 def parallel(self, dist='b', block=None, **flags):
359 359 """Decorator for making a ParallelFunction"""
360 360 block = self.block if block is None else block
361 361 return parallel(self, dist=dist, block=block, **flags)
362 362
363 363 @testdec.skip_doctest
364 364 class DirectView(View):
365 365 """Direct Multiplexer View of one or more engines.
366 366
367 367 These are created via indexed access to a client:
368 368
369 369 >>> dv_1 = client[1]
370 370 >>> dv_all = client[:]
371 371 >>> dv_even = client[::2]
372 372 >>> dv_some = client[1:3]
373 373
374 374 This object provides dictionary access to engine namespaces:
375 375
376 376 # push a=5:
377 377 >>> dv['a'] = 5
378 378 # pull 'foo':
379 379 >>> db['foo']
380 380
381 381 """
382 382
383 383 def __init__(self, client=None, socket=None, targets=None):
384 384 super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
385 385
386 386 @property
387 387 def importer(self):
388 388 """sync_imports(local=True) as a property.
389 389
390 390 See sync_imports for details.
391 391
392 392 """
393 393 return self.sync_imports(True)
394 394
395 395 @contextmanager
396 396 def sync_imports(self, local=True):
397 397 """Context Manager for performing simultaneous local and remote imports.
398 398
399 399 'import x as y' will *not* work. The 'as y' part will simply be ignored.
400 400
401 401 >>> with view.sync_imports():
402 402 ... from numpy import recarray
403 403 importing recarray from numpy on engine(s)
404 404
405 405 """
406 406 import __builtin__
407 407 local_import = __builtin__.__import__
408 408 modules = set()
409 409 results = []
410 410 @util.interactive
411 411 def remote_import(name, fromlist, level):
412 412 """the function to be passed to apply, that actually performs the import
413 413 on the engine, and loads up the user namespace.
414 414 """
415 415 import sys
416 416 user_ns = globals()
417 417 mod = __import__(name, fromlist=fromlist, level=level)
418 418 if fromlist:
419 419 for key in fromlist:
420 420 user_ns[key] = getattr(mod, key)
421 421 else:
422 422 user_ns[name] = sys.modules[name]
423 423
424 424 def view_import(name, globals={}, locals={}, fromlist=[], level=-1):
425 425 """the drop-in replacement for __import__, that optionally imports
426 426 locally as well.
427 427 """
428 428 # don't override nested imports
429 429 save_import = __builtin__.__import__
430 430 __builtin__.__import__ = local_import
431 431
432 432 if imp.lock_held():
433 433 # this is a side-effect import, don't do it remotely, or even
434 434 # ignore the local effects
435 435 return local_import(name, globals, locals, fromlist, level)
436 436
437 437 imp.acquire_lock()
438 438 if local:
439 439 mod = local_import(name, globals, locals, fromlist, level)
440 440 else:
441 441 raise NotImplementedError("remote-only imports not yet implemented")
442 442 imp.release_lock()
443 443
444 444 key = name+':'+','.join(fromlist or [])
445 445 if level == -1 and key not in modules:
446 446 modules.add(key)
447 447 if fromlist:
448 448 print "importing %s from %s on engine(s)"%(','.join(fromlist), name)
449 449 else:
450 450 print "importing %s on engine(s)"%name
451 451 results.append(self.apply_async(remote_import, name, fromlist, level))
452 452 # restore override
453 453 __builtin__.__import__ = save_import
454 454
455 455 return mod
456 456
457 457 # override __import__
458 458 __builtin__.__import__ = view_import
459 459 try:
460 460 # enter the block
461 461 yield
462 462 except ImportError:
463 463 if not local:
464 464 # ignore import errors if not doing local imports
465 465 pass
466 466 finally:
467 467 # always restore __import__
468 468 __builtin__.__import__ = local_import
469 469
470 470 for r in results:
471 471 # raise possible remote ImportErrors here
472 472 r.get()
473 473
474 474
475 475 @sync_results
476 476 @save_ids
477 477 def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
478 478 """calls f(*args, **kwargs) on remote engines, returning the result.
479 479
480 480 This method sets all of `apply`'s flags via this View's attributes.
481 481
482 482 Parameters
483 483 ----------
484 484
485 485 f : callable
486 486
487 487 args : list [default: empty]
488 488
489 489 kwargs : dict [default: empty]
490 490
491 491 targets : target list [default: self.targets]
492 492 where to run
493 493 block : bool [default: self.block]
494 494 whether to block
495 495 track : bool [default: self.track]
496 496 whether to ask zmq to track the message, for safe non-copying sends
497 497
498 498 Returns
499 499 -------
500 500
501 501 if self.block is False:
502 502 returns AsyncResult
503 503 else:
504 504 returns actual result of f(*args, **kwargs) on the engine(s)
505 505 This will be a list of self.targets is also a list (even length 1), or
506 506 the single result if self.targets is an integer engine id
507 507 """
508 508 args = [] if args is None else args
509 509 kwargs = {} if kwargs is None else kwargs
510 510 block = self.block if block is None else block
511 511 track = self.track if track is None else track
512 512 targets = self.targets if targets is None else targets
513 513
514 514 _idents = self.client._build_targets(targets)[0]
515 515 msg_ids = []
516 516 trackers = []
517 517 for ident in _idents:
518 518 msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
519 519 ident=ident)
520 520 if track:
521 521 trackers.append(msg['tracker'])
522 522 msg_ids.append(msg['msg_id'])
523 523 tracker = None if track is False else zmq.MessageTracker(*trackers)
524 524 ar = AsyncResult(self.client, msg_ids, fname=f.__name__, targets=targets, tracker=tracker)
525 525 if block:
526 526 try:
527 527 return ar.get()
528 528 except KeyboardInterrupt:
529 529 pass
530 530 return ar
531 531
532 532 @spin_after
533 533 def map(self, f, *sequences, **kwargs):
534 534 """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
535 535
536 536 Parallel version of builtin `map`, using this View's `targets`.
537 537
538 538 There will be one task per target, so work will be chunked
539 539 if the sequences are longer than `targets`.
540 540
541 541 Results can be iterated as they are ready, but will become available in chunks.
542 542
543 543 Parameters
544 544 ----------
545 545
546 546 f : callable
547 547 function to be mapped
548 548 *sequences: one or more sequences of matching length
549 549 the sequences to be distributed and passed to `f`
550 550 block : bool
551 551 whether to wait for the result or not [default self.block]
552 552
553 553 Returns
554 554 -------
555 555
556 556 if block=False:
557 557 AsyncMapResult
558 558 An object like AsyncResult, but which reassembles the sequence of results
559 559 into a single list. AsyncMapResults can be iterated through before all
560 560 results are complete.
561 561 else:
562 562 list
563 563 the result of map(f,*sequences)
564 564 """
565 565
566 566 block = kwargs.pop('block', self.block)
567 567 for k in kwargs.keys():
568 568 if k not in ['block', 'track']:
569 569 raise TypeError("invalid keyword arg, %r"%k)
570 570
571 571 assert len(sequences) > 0, "must have some sequences to map onto!"
572 572 pf = ParallelFunction(self, f, block=block, **kwargs)
573 573 return pf.map(*sequences)
574 574
575 575 def execute(self, code, targets=None, block=None):
576 576 """Executes `code` on `targets` in blocking or nonblocking manner.
577 577
578 578 ``execute`` is always `bound` (affects engine namespace)
579 579
580 580 Parameters
581 581 ----------
582 582
583 583 code : str
584 584 the code string to be executed
585 585 block : bool
586 586 whether or not to wait until done to return
587 587 default: self.block
588 588 """
589 589 return self._really_apply(util._execute, args=(code,), block=block, targets=targets)
590 590
591 591 def run(self, filename, targets=None, block=None):
592 592 """Execute contents of `filename` on my engine(s).
593 593
594 594 This simply reads the contents of the file and calls `execute`.
595 595
596 596 Parameters
597 597 ----------
598 598
599 599 filename : str
600 600 The path to the file
601 601 targets : int/str/list of ints/strs
602 602 the engines on which to execute
603 603 default : all
604 604 block : bool
605 605 whether or not to wait until done
606 606 default: self.block
607 607
608 608 """
609 609 with open(filename, 'r') as f:
610 610 # add newline in case of trailing indented whitespace
611 611 # which will cause SyntaxError
612 612 code = f.read()+'\n'
613 613 return self.execute(code, block=block, targets=targets)
614 614
615 615 def update(self, ns):
616 616 """update remote namespace with dict `ns`
617 617
618 618 See `push` for details.
619 619 """
620 620 return self.push(ns, block=self.block, track=self.track)
621 621
622 622 def push(self, ns, targets=None, block=None, track=None):
623 623 """update remote namespace with dict `ns`
624 624
625 625 Parameters
626 626 ----------
627 627
628 628 ns : dict
629 629 dict of keys with which to update engine namespace(s)
630 630 block : bool [default : self.block]
631 631 whether to wait to be notified of engine receipt
632 632
633 633 """
634 634
635 635 block = block if block is not None else self.block
636 636 track = track if track is not None else self.track
637 637 targets = targets if targets is not None else self.targets
638 638 # applier = self.apply_sync if block else self.apply_async
639 639 if not isinstance(ns, dict):
640 640 raise TypeError("Must be a dict, not %s"%type(ns))
641 641 return self._really_apply(util._push, (ns,), block=block, track=track, targets=targets)
642 642
643 643 def get(self, key_s):
644 644 """get object(s) by `key_s` from remote namespace
645 645
646 646 see `pull` for details.
647 647 """
648 648 # block = block if block is not None else self.block
649 649 return self.pull(key_s, block=True)
650 650
651 651 def pull(self, names, targets=None, block=None):
652 652 """get object(s) by `name` from remote namespace
653 653
654 654 will return one object if it is a key.
655 655 can also take a list of keys, in which case it will return a list of objects.
656 656 """
657 657 block = block if block is not None else self.block
658 658 targets = targets if targets is not None else self.targets
659 659 applier = self.apply_sync if block else self.apply_async
660 660 if isinstance(names, basestring):
661 661 pass
662 662 elif isinstance(names, (list,tuple,set)):
663 663 for key in names:
664 664 if not isinstance(key, basestring):
665 665 raise TypeError("keys must be str, not type %r"%type(key))
666 666 else:
667 667 raise TypeError("names must be strs, not %r"%names)
668 668 return self._really_apply(util._pull, (names,), block=block, targets=targets)
669 669
670 670 def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
671 671 """
672 672 Partition a Python sequence and send the partitions to a set of engines.
673 673 """
674 674 block = block if block is not None else self.block
675 675 track = track if track is not None else self.track
676 676 targets = targets if targets is not None else self.targets
677 677
678 678 mapObject = Map.dists[dist]()
679 679 nparts = len(targets)
680 680 msg_ids = []
681 681 trackers = []
682 682 for index, engineid in enumerate(targets):
683 683 partition = mapObject.getPartition(seq, index, nparts)
684 684 if flatten and len(partition) == 1:
685 685 ns = {key: partition[0]}
686 686 else:
687 687 ns = {key: partition}
688 688 r = self.push(ns, block=False, track=track, targets=engineid)
689 689 msg_ids.extend(r.msg_ids)
690 690 if track:
691 691 trackers.append(r._tracker)
692 692
693 693 if track:
694 694 tracker = zmq.MessageTracker(*trackers)
695 695 else:
696 696 tracker = None
697 697
698 698 r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
699 699 if block:
700 700 r.wait()
701 701 else:
702 702 return r
703 703
704 704 @sync_results
705 705 @save_ids
706 706 def gather(self, key, dist='b', targets=None, block=None):
707 707 """
708 708 Gather a partitioned sequence on a set of engines as a single local seq.
709 709 """
710 710 block = block if block is not None else self.block
711 711 targets = targets if targets is not None else self.targets
712 712 mapObject = Map.dists[dist]()
713 713 msg_ids = []
714 714
715 715 for index, engineid in enumerate(targets):
716 716 msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
717 717
718 718 r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
719 719
720 720 if block:
721 721 try:
722 722 return r.get()
723 723 except KeyboardInterrupt:
724 724 pass
725 725 return r
726 726
727 727 def __getitem__(self, key):
728 728 return self.get(key)
729 729
730 730 def __setitem__(self,key, value):
731 731 self.update({key:value})
732 732
733 733 def clear(self, targets=None, block=False):
734 734 """Clear the remote namespaces on my engines."""
735 735 block = block if block is not None else self.block
736 736 targets = targets if targets is not None else self.targets
737 737 return self.client.clear(targets=targets, block=block)
738 738
739 739 def kill(self, targets=None, block=True):
740 740 """Kill my engines."""
741 741 block = block if block is not None else self.block
742 742 targets = targets if targets is not None else self.targets
743 743 return self.client.kill(targets=targets, block=block)
744 744
745 745 #----------------------------------------
746 746 # activate for %px,%autopx magics
747 747 #----------------------------------------
748 748 def activate(self):
749 749 """Make this `View` active for parallel magic commands.
750 750
751 751 IPython has a magic command syntax to work with `MultiEngineClient` objects.
752 752 In a given IPython session there is a single active one. While
753 753 there can be many `Views` created and used by the user,
754 754 there is only one active one. The active `View` is used whenever
755 755 the magic commands %px and %autopx are used.
756 756
757 757 The activate() method is called on a given `View` to make it
758 758 active. Once this has been done, the magic commands can be used.
759 759 """
760 760
761 761 try:
762 762 # This is injected into __builtins__.
763 763 ip = get_ipython()
764 764 except NameError:
765 765 print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
766 766 else:
767 767 pmagic = ip.plugin_manager.get_plugin('parallelmagic')
768 if pmagic is not None:
769 pmagic.active_view = self
770 else:
771 print "You must first load the parallelmagic extension " \
772 "by doing '%load_ext parallelmagic'"
768 if pmagic is None:
769 ip.magic_load_ext('parallelmagic')
770 pmagic = ip.plugin_manager.get_plugin('parallelmagic')
771
772 pmagic.active_view = self
773 773
774 774
775 775 @testdec.skip_doctest
776 776 class LoadBalancedView(View):
777 777 """An load-balancing View that only executes via the Task scheduler.
778 778
779 779 Load-balanced views can be created with the client's `view` method:
780 780
781 781 >>> v = client.load_balanced_view()
782 782
783 783 or targets can be specified, to restrict the potential destinations:
784 784
785 785 >>> v = client.client.load_balanced_view(([1,3])
786 786
787 787 which would restrict loadbalancing to between engines 1 and 3.
788 788
789 789 """
790 790
791 791 follow=Any()
792 792 after=Any()
793 793 timeout=CFloat()
794 794
795 795 _task_scheme = Any()
796 796 _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout'])
797 797
798 798 def __init__(self, client=None, socket=None, **flags):
799 799 super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
800 800 self._task_scheme=client._task_scheme
801 801
802 802 def _validate_dependency(self, dep):
803 803 """validate a dependency.
804 804
805 805 For use in `set_flags`.
806 806 """
807 807 if dep is None or isinstance(dep, (str, AsyncResult, Dependency)):
808 808 return True
809 809 elif isinstance(dep, (list,set, tuple)):
810 810 for d in dep:
811 811 if not isinstance(d, (str, AsyncResult)):
812 812 return False
813 813 elif isinstance(dep, dict):
814 814 if set(dep.keys()) != set(Dependency().as_dict().keys()):
815 815 return False
816 816 if not isinstance(dep['msg_ids'], list):
817 817 return False
818 818 for d in dep['msg_ids']:
819 819 if not isinstance(d, str):
820 820 return False
821 821 else:
822 822 return False
823 823
824 824 return True
825 825
826 826 def _render_dependency(self, dep):
827 827 """helper for building jsonable dependencies from various input forms."""
828 828 if isinstance(dep, Dependency):
829 829 return dep.as_dict()
830 830 elif isinstance(dep, AsyncResult):
831 831 return dep.msg_ids
832 832 elif dep is None:
833 833 return []
834 834 else:
835 835 # pass to Dependency constructor
836 836 return list(Dependency(dep))
837 837
838 838 def set_flags(self, **kwargs):
839 839 """set my attribute flags by keyword.
840 840
841 841 A View is a wrapper for the Client's apply method, but with attributes
842 842 that specify keyword arguments, those attributes can be set by keyword
843 843 argument with this method.
844 844
845 845 Parameters
846 846 ----------
847 847
848 848 block : bool
849 849 whether to wait for results
850 850 track : bool
851 851 whether to create a MessageTracker to allow the user to
852 852 safely edit after arrays and buffers during non-copying
853 853 sends.
854 854 #
855 855 after : Dependency or collection of msg_ids
856 856 Only for load-balanced execution (targets=None)
857 857 Specify a list of msg_ids as a time-based dependency.
858 858 This job will only be run *after* the dependencies
859 859 have been met.
860 860
861 861 follow : Dependency or collection of msg_ids
862 862 Only for load-balanced execution (targets=None)
863 863 Specify a list of msg_ids as a location-based dependency.
864 864 This job will only be run on an engine where this dependency
865 865 is met.
866 866
867 867 timeout : float/int or None
868 868 Only for load-balanced execution (targets=None)
869 869 Specify an amount of time (in seconds) for the scheduler to
870 870 wait for dependencies to be met before failing with a
871 871 DependencyTimeout.
872 872 """
873 873
874 874 super(LoadBalancedView, self).set_flags(**kwargs)
875 875 for name in ('follow', 'after'):
876 876 if name in kwargs:
877 877 value = kwargs[name]
878 878 if self._validate_dependency(value):
879 879 setattr(self, name, value)
880 880 else:
881 881 raise ValueError("Invalid dependency: %r"%value)
882 882 if 'timeout' in kwargs:
883 883 t = kwargs['timeout']
884 884 if not isinstance(t, (int, long, float, type(None))):
885 885 raise TypeError("Invalid type for timeout: %r"%type(t))
886 886 if t is not None:
887 887 if t < 0:
888 888 raise ValueError("Invalid timeout: %s"%t)
889 889 self.timeout = t
890 890
891 891 @sync_results
892 892 @save_ids
893 893 def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
894 894 after=None, follow=None, timeout=None,
895 895 targets=None):
896 896 """calls f(*args, **kwargs) on a remote engine, returning the result.
897 897
898 898 This method temporarily sets all of `apply`'s flags for a single call.
899 899
900 900 Parameters
901 901 ----------
902 902
903 903 f : callable
904 904
905 905 args : list [default: empty]
906 906
907 907 kwargs : dict [default: empty]
908 908
909 909 block : bool [default: self.block]
910 910 whether to block
911 911 track : bool [default: self.track]
912 912 whether to ask zmq to track the message, for safe non-copying sends
913 913
914 914 !!!!!! TODO: THE REST HERE !!!!
915 915
916 916 Returns
917 917 -------
918 918
919 919 if self.block is False:
920 920 returns AsyncResult
921 921 else:
922 922 returns actual result of f(*args, **kwargs) on the engine(s)
923 923 This will be a list of self.targets is also a list (even length 1), or
924 924 the single result if self.targets is an integer engine id
925 925 """
926 926
927 927 # validate whether we can run
928 928 if self._socket.closed:
929 929 msg = "Task farming is disabled"
930 930 if self._task_scheme == 'pure':
931 931 msg += " because the pure ZMQ scheduler cannot handle"
932 932 msg += " disappearing engines."
933 933 raise RuntimeError(msg)
934 934
935 935 if self._task_scheme == 'pure':
936 936 # pure zmq scheme doesn't support dependencies
937 937 msg = "Pure ZMQ scheduler doesn't support dependencies"
938 938 if (follow or after):
939 939 # hard fail on DAG dependencies
940 940 raise RuntimeError(msg)
941 941 if isinstance(f, dependent):
942 942 # soft warn on functional dependencies
943 943 warnings.warn(msg, RuntimeWarning)
944 944
945 945 # build args
946 946 args = [] if args is None else args
947 947 kwargs = {} if kwargs is None else kwargs
948 948 block = self.block if block is None else block
949 949 track = self.track if track is None else track
950 950 after = self.after if after is None else after
951 951 follow = self.follow if follow is None else follow
952 952 timeout = self.timeout if timeout is None else timeout
953 953 targets = self.targets if targets is None else targets
954 954
955 955 if targets is None:
956 956 idents = []
957 957 else:
958 958 idents = self.client._build_targets(targets)[0]
959 959
960 960 after = self._render_dependency(after)
961 961 follow = self._render_dependency(follow)
962 962 subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents)
963 963
964 964 msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track,
965 965 subheader=subheader)
966 966 tracker = None if track is False else msg['tracker']
967 967
968 968 ar = AsyncResult(self.client, msg['msg_id'], fname=f.__name__, targets=None, tracker=tracker)
969 969
970 970 if block:
971 971 try:
972 972 return ar.get()
973 973 except KeyboardInterrupt:
974 974 pass
975 975 return ar
976 976
977 977 @spin_after
978 978 @save_ids
979 979 def map(self, f, *sequences, **kwargs):
980 980 """view.map(f, *sequences, block=self.block, chunksize=1) => list|AsyncMapResult
981 981
982 982 Parallel version of builtin `map`, load-balanced by this View.
983 983
984 984 `block`, and `chunksize` can be specified by keyword only.
985 985
986 986 Each `chunksize` elements will be a separate task, and will be
987 987 load-balanced. This lets individual elements be available for iteration
988 988 as soon as they arrive.
989 989
990 990 Parameters
991 991 ----------
992 992
993 993 f : callable
994 994 function to be mapped
995 995 *sequences: one or more sequences of matching length
996 996 the sequences to be distributed and passed to `f`
997 997 block : bool
998 998 whether to wait for the result or not [default self.block]
999 999 track : bool
1000 1000 whether to create a MessageTracker to allow the user to
1001 1001 safely edit after arrays and buffers during non-copying
1002 1002 sends.
1003 1003 chunksize : int
1004 1004 how many elements should be in each task [default 1]
1005 1005
1006 1006 Returns
1007 1007 -------
1008 1008
1009 1009 if block=False:
1010 1010 AsyncMapResult
1011 1011 An object like AsyncResult, but which reassembles the sequence of results
1012 1012 into a single list. AsyncMapResults can be iterated through before all
1013 1013 results are complete.
1014 1014 else:
1015 1015 the result of map(f,*sequences)
1016 1016
1017 1017 """
1018 1018
1019 1019 # default
1020 1020 block = kwargs.get('block', self.block)
1021 1021 chunksize = kwargs.get('chunksize', 1)
1022 1022
1023 1023 keyset = set(kwargs.keys())
1024 1024 extra_keys = keyset.difference_update(set(['block', 'chunksize']))
1025 1025 if extra_keys:
1026 1026 raise TypeError("Invalid kwargs: %s"%list(extra_keys))
1027 1027
1028 1028 assert len(sequences) > 0, "must have some sequences to map onto!"
1029 1029
1030 1030 pf = ParallelFunction(self, f, block=block, chunksize=chunksize)
1031 1031 return pf.map(*sequences)
1032 1032
1033 1033 __all__ = ['LoadBalancedView', 'DirectView'] No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now