##// END OF EJS Templates
Fix remaining test failures....
Fernando Perez -
Show More
@@ -1,3820 +1,3822 b''
1 1 # encoding: utf-8
2 2 """Magic functions for InteractiveShell.
3 3 """
4 4
5 5 #-----------------------------------------------------------------------------
6 6 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
7 7 # Copyright (C) 2001-2007 Fernando Perez <fperez@colorado.edu>
8 8 # Copyright (C) 2008-2011 The IPython Development Team
9 9
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import __builtin__ as builtin_mod
19 19 import __future__
20 20 import bdb
21 21 import inspect
22 22 import io
23 23 import json
24 24 import os
25 25 import sys
26 26 import re
27 27 import time
28 28 import gc
29 29 from StringIO import StringIO
30 30 from getopt import getopt,GetoptError
31 31 from pprint import pformat
32 32 from urllib2 import urlopen
33 33
34 34 # cProfile was added in Python2.5
35 35 try:
36 36 import cProfile as profile
37 37 import pstats
38 38 except ImportError:
39 39 # profile isn't bundled by default in Debian for license reasons
40 40 try:
41 41 import profile,pstats
42 42 except ImportError:
43 43 profile = pstats = None
44 44
45 45 from IPython.core import debugger, oinspect
46 46 from IPython.core.error import TryNext
47 47 from IPython.core.error import UsageError
48 48 from IPython.core.error import StdinNotImplementedError
49 49 from IPython.core.macro import Macro
50 50 from IPython.core import magic_arguments, page
51 51 from IPython.core.prefilter import ESC_MAGIC
52 52 from IPython.testing.skipdoctest import skip_doctest
53 53 from IPython.utils import py3compat
54 54 from IPython.utils.encoding import DEFAULT_ENCODING
55 55 from IPython.utils.io import file_read, nlprint
56 56 from IPython.utils.module_paths import find_mod
57 57 from IPython.utils.path import get_py_filename, unquote_filename
58 58 from IPython.utils.process import arg_split, abbrev_cwd
59 59 from IPython.utils.terminal import set_term_title
60 60 from IPython.utils.text import format_screen
61 61 from IPython.utils.timing import clock, clock2
62 62 from IPython.utils.warn import warn, error
63 63 from IPython.utils.ipstruct import Struct
64 64 from IPython.config.application import Application
65 65
66 66 #-----------------------------------------------------------------------------
67 67 # Utility functions
68 68 #-----------------------------------------------------------------------------
69 69
70 70 def on_off(tag):
71 71 """Return an ON/OFF string for a 1/0 input. Simple utility function."""
72 72 return ['OFF','ON'][tag]
73 73
74 74 class Bunch: pass
75 75
76 76 def compress_dhist(dh):
77 77 head, tail = dh[:-10], dh[-10:]
78 78
79 79 newhead = []
80 80 done = set()
81 81 for h in head:
82 82 if h in done:
83 83 continue
84 84 newhead.append(h)
85 85 done.add(h)
86 86
87 87 return newhead + tail
88 88
89 89 def needs_local_scope(func):
90 90 """Decorator to mark magic functions which need to local scope to run."""
91 91 func.needs_local_scope = True
92 92 return func
93 93
94 94
95 95 # Used for exception handling in magic_edit
96 96 class MacroToEdit(ValueError): pass
97 97
98 98 #***************************************************************************
99 99 # Main class implementing Magic functionality
100 100
101 101 # XXX - for some odd reason, if Magic is made a new-style class, we get errors
102 102 # on construction of the main InteractiveShell object. Something odd is going
103 103 # on with super() calls, Configurable and the MRO... For now leave it as-is, but
104 104 # eventually this needs to be clarified.
105 105 # BG: This is because InteractiveShell inherits from this, but is itself a
106 106 # Configurable. This messes up the MRO in some way. The fix is that we need to
107 107 # make Magic a configurable that InteractiveShell does not subclass.
108 108
109 109 class Magic(object):
110 110 """Magic functions for InteractiveShell.
111 111
112 112 Shell functions which can be reached as %function_name. All magic
113 113 functions should accept a string, which they can parse for their own
114 114 needs. This can make some functions easier to type, eg `%cd ../`
115 115 vs. `%cd("../")`
116 116
117 117 ALL definitions MUST begin with the prefix magic_. The user won't need it
118 118 at the command line, but it is is needed in the definition. """
119 119
120 120 # class globals
121 121 auto_status = ['Automagic is OFF, % prefix IS needed for magic functions.',
122 122 'Automagic is ON, % prefix NOT needed for magic functions.']
123 123
124 124
125 125 configurables = None
126 126
127 127 default_runner = None
128 128 #......................................................................
129 129 # some utility functions
130 130
131 131 def __init__(self, shell):
132 132
133 133 self.options_table = {}
134 134 if profile is None:
135 135 self.magic_prun = self.profile_missing_notice
136 136 self.shell = shell
137 137 if self.configurables is None:
138 138 self.configurables = []
139 139
140 140 # namespace for holding state we may need
141 141 self._magic_state = Bunch()
142 142
143 143 def profile_missing_notice(self, *args, **kwargs):
144 144 error("""\
145 145 The profile module could not be found. It has been removed from the standard
146 146 python packages because of its non-free license. To use profiling, install the
147 147 python-profiler package from non-free.""")
148 148
149 149 def default_option(self,fn,optstr):
150 150 """Make an entry in the options_table for fn, with value optstr"""
151 151
152 152 if fn not in self.lsmagic():
153 153 error("%s is not a magic function" % fn)
154 154 self.options_table[fn] = optstr
155 155
156 156 def lsmagic(self):
157 157 """Return a list of currently available magic functions.
158 158
159 159 Gives a list of the bare names after mangling (['ls','cd', ...], not
160 160 ['magic_ls','magic_cd',...]"""
161 161
162 162 # FIXME. This needs a cleanup, in the way the magics list is built.
163 163
164 164 # magics in class definition
165 165 class_magic = lambda fn: fn.startswith('magic_') and \
166 166 callable(Magic.__dict__[fn])
167 167 # in instance namespace (run-time user additions)
168 168 inst_magic = lambda fn: fn.startswith('magic_') and \
169 169 callable(self.__dict__[fn])
170 170 # and bound magics by user (so they can access self):
171 171 inst_bound_magic = lambda fn: fn.startswith('magic_') and \
172 172 callable(self.__class__.__dict__[fn])
173 173 magics = filter(class_magic,Magic.__dict__.keys()) + \
174 174 filter(inst_magic, self.__dict__.keys()) + \
175 175 filter(inst_bound_magic, self.__class__.__dict__.keys())
176 176 out = []
177 177 for fn in set(magics):
178 178 out.append(fn.replace('magic_','',1))
179 179 out.sort()
180 180 return out
181 181
182 182 def arg_err(self,func):
183 183 """Print docstring if incorrect arguments were passed"""
184 184 print 'Error in arguments:'
185 185 print oinspect.getdoc(func)
186 186
187 187 def format_latex(self,strng):
188 188 """Format a string for latex inclusion."""
189 189
190 190 # Characters that need to be escaped for latex:
191 191 escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
192 192 # Magic command names as headers:
193 193 cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
194 194 re.MULTILINE)
195 195 # Magic commands
196 196 cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
197 197 re.MULTILINE)
198 198 # Paragraph continue
199 199 par_re = re.compile(r'\\$',re.MULTILINE)
200 200
201 201 # The "\n" symbol
202 202 newline_re = re.compile(r'\\n')
203 203
204 204 # Now build the string for output:
205 205 #strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
206 206 strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
207 207 strng)
208 208 strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
209 209 strng = par_re.sub(r'\\\\',strng)
210 210 strng = escape_re.sub(r'\\\1',strng)
211 211 strng = newline_re.sub(r'\\textbackslash{}n',strng)
212 212 return strng
213 213
214 214 def parse_options(self,arg_str,opt_str,*long_opts,**kw):
215 215 """Parse options passed to an argument string.
216 216
217 217 The interface is similar to that of getopt(), but it returns back a
218 218 Struct with the options as keys and the stripped argument string still
219 219 as a string.
220 220
221 221 arg_str is quoted as a true sys.argv vector by using shlex.split.
222 222 This allows us to easily expand variables, glob files, quote
223 223 arguments, etc.
224 224
225 225 Options:
226 226 -mode: default 'string'. If given as 'list', the argument string is
227 227 returned as a list (split on whitespace) instead of a string.
228 228
229 229 -list_all: put all option values in lists. Normally only options
230 230 appearing more than once are put in a list.
231 231
232 232 -posix (True): whether to split the input line in POSIX mode or not,
233 233 as per the conventions outlined in the shlex module from the
234 234 standard library."""
235 235
236 236 # inject default options at the beginning of the input line
237 237 caller = sys._getframe(1).f_code.co_name.replace('magic_','')
238 238 arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
239 239
240 240 mode = kw.get('mode','string')
241 241 if mode not in ['string','list']:
242 242 raise ValueError,'incorrect mode given: %s' % mode
243 243 # Get options
244 244 list_all = kw.get('list_all',0)
245 245 posix = kw.get('posix', os.name == 'posix')
246 246 strict = kw.get('strict', True)
247 247
248 248 # Check if we have more than one argument to warrant extra processing:
249 249 odict = {} # Dictionary with options
250 250 args = arg_str.split()
251 251 if len(args) >= 1:
252 252 # If the list of inputs only has 0 or 1 thing in it, there's no
253 253 # need to look for options
254 254 argv = arg_split(arg_str, posix, strict)
255 255 # Do regular option processing
256 256 try:
257 257 opts,args = getopt(argv,opt_str,*long_opts)
258 258 except GetoptError,e:
259 259 raise UsageError('%s ( allowed: "%s" %s)' % (e.msg,opt_str,
260 260 " ".join(long_opts)))
261 261 for o,a in opts:
262 262 if o.startswith('--'):
263 263 o = o[2:]
264 264 else:
265 265 o = o[1:]
266 266 try:
267 267 odict[o].append(a)
268 268 except AttributeError:
269 269 odict[o] = [odict[o],a]
270 270 except KeyError:
271 271 if list_all:
272 272 odict[o] = [a]
273 273 else:
274 274 odict[o] = a
275 275
276 276 # Prepare opts,args for return
277 277 opts = Struct(odict)
278 278 if mode == 'string':
279 279 args = ' '.join(args)
280 280
281 281 return opts,args
282 282
283 283 #......................................................................
284 284 # And now the actual magic functions
285 285
286 286 # Functions for IPython shell work (vars,funcs, config, etc)
287 287 def magic_lsmagic(self, parameter_s = ''):
288 288 """List currently available magic functions."""
289 289 mesc = ESC_MAGIC
290 290 print 'Available magic functions:\n'+mesc+\
291 291 (' '+mesc).join(self.lsmagic())
292 292 print '\n' + Magic.auto_status[self.shell.automagic]
293 293 return None
294 294
295 295 def magic_magic(self, parameter_s = ''):
296 296 """Print information about the magic function system.
297 297
298 298 Supported formats: -latex, -brief, -rest
299 299 """
300 300
301 301 mode = ''
302 302 try:
303 303 if parameter_s.split()[0] == '-latex':
304 304 mode = 'latex'
305 305 if parameter_s.split()[0] == '-brief':
306 306 mode = 'brief'
307 307 if parameter_s.split()[0] == '-rest':
308 308 mode = 'rest'
309 309 rest_docs = []
310 310 except:
311 311 pass
312 312
313 313 magic_docs = []
314 314 for fname in self.lsmagic():
315 315 mname = 'magic_' + fname
316 316 for space in (Magic, self, self.__class__):
317 317 try:
318 318 fn = space.__dict__[mname]
319 319 except KeyError:
320 320 pass
321 321 else:
322 322 break
323 323 if mode == 'brief':
324 324 # only first line
325 325 if fn.__doc__:
326 326 fndoc = fn.__doc__.split('\n',1)[0]
327 327 else:
328 328 fndoc = 'No documentation'
329 329 else:
330 330 if fn.__doc__:
331 331 fndoc = fn.__doc__.rstrip()
332 332 else:
333 333 fndoc = 'No documentation'
334 334
335 335
336 336 if mode == 'rest':
337 337 rest_docs.append('**%s%s**::\n\n\t%s\n\n' %(ESC_MAGIC,
338 338 fname,fndoc))
339 339
340 340 else:
341 341 magic_docs.append('%s%s:\n\t%s\n' %(ESC_MAGIC,
342 342 fname,fndoc))
343 343
344 344 magic_docs = ''.join(magic_docs)
345 345
346 346 if mode == 'rest':
347 347 return "".join(rest_docs)
348 348
349 349 if mode == 'latex':
350 350 print self.format_latex(magic_docs)
351 351 return
352 352 else:
353 353 magic_docs = format_screen(magic_docs)
354 354 if mode == 'brief':
355 355 return magic_docs
356 356
357 357 outmsg = """
358 358 IPython's 'magic' functions
359 359 ===========================
360 360
361 361 The magic function system provides a series of functions which allow you to
362 362 control the behavior of IPython itself, plus a lot of system-type
363 363 features. All these functions are prefixed with a % character, but parameters
364 364 are given without parentheses or quotes.
365 365
366 366 NOTE: If you have 'automagic' enabled (via the command line option or with the
367 367 %automagic function), you don't need to type in the % explicitly. By default,
368 368 IPython ships with automagic on, so you should only rarely need the % escape.
369 369
370 370 Example: typing '%cd mydir' (without the quotes) changes you working directory
371 371 to 'mydir', if it exists.
372 372
373 373 For a list of the available magic functions, use %lsmagic. For a description
374 374 of any of them, type %magic_name?, e.g. '%cd?'.
375 375
376 376 Currently the magic system has the following functions:\n"""
377 377
378 378 mesc = ESC_MAGIC
379 379 outmsg = ("%s\n%s\n\nSummary of magic functions (from %slsmagic):"
380 380 "\n\n%s%s\n\n%s" % (outmsg,
381 381 magic_docs,mesc,mesc,
382 382 (' '+mesc).join(self.lsmagic()),
383 383 Magic.auto_status[self.shell.automagic] ) )
384 384 page.page(outmsg)
385 385
386 386 def magic_automagic(self, parameter_s = ''):
387 387 """Make magic functions callable without having to type the initial %.
388 388
389 389 Without argumentsl toggles on/off (when off, you must call it as
390 390 %automagic, of course). With arguments it sets the value, and you can
391 391 use any of (case insensitive):
392 392
393 393 - on,1,True: to activate
394 394
395 395 - off,0,False: to deactivate.
396 396
397 397 Note that magic functions have lowest priority, so if there's a
398 398 variable whose name collides with that of a magic fn, automagic won't
399 399 work for that function (you get the variable instead). However, if you
400 400 delete the variable (del var), the previously shadowed magic function
401 401 becomes visible to automagic again."""
402 402
403 403 arg = parameter_s.lower()
404 404 if parameter_s in ('on','1','true'):
405 405 self.shell.automagic = True
406 406 elif parameter_s in ('off','0','false'):
407 407 self.shell.automagic = False
408 408 else:
409 409 self.shell.automagic = not self.shell.automagic
410 410 print '\n' + Magic.auto_status[self.shell.automagic]
411 411
412 412 @skip_doctest
413 413 def magic_autocall(self, parameter_s = ''):
414 414 """Make functions callable without having to type parentheses.
415 415
416 416 Usage:
417 417
418 418 %autocall [mode]
419 419
420 420 The mode can be one of: 0->Off, 1->Smart, 2->Full. If not given, the
421 421 value is toggled on and off (remembering the previous state).
422 422
423 423 In more detail, these values mean:
424 424
425 425 0 -> fully disabled
426 426
427 427 1 -> active, but do not apply if there are no arguments on the line.
428 428
429 429 In this mode, you get::
430 430
431 431 In [1]: callable
432 432 Out[1]: <built-in function callable>
433 433
434 434 In [2]: callable 'hello'
435 435 ------> callable('hello')
436 436 Out[2]: False
437 437
438 438 2 -> Active always. Even if no arguments are present, the callable
439 439 object is called::
440 440
441 441 In [2]: float
442 442 ------> float()
443 443 Out[2]: 0.0
444 444
445 445 Note that even with autocall off, you can still use '/' at the start of
446 446 a line to treat the first argument on the command line as a function
447 447 and add parentheses to it::
448 448
449 449 In [8]: /str 43
450 450 ------> str(43)
451 451 Out[8]: '43'
452 452
453 453 # all-random (note for auto-testing)
454 454 """
455 455
456 456 if parameter_s:
457 457 arg = int(parameter_s)
458 458 else:
459 459 arg = 'toggle'
460 460
461 461 if not arg in (0,1,2,'toggle'):
462 462 error('Valid modes: (0->Off, 1->Smart, 2->Full')
463 463 return
464 464
465 465 if arg in (0,1,2):
466 466 self.shell.autocall = arg
467 467 else: # toggle
468 468 if self.shell.autocall:
469 469 self._magic_state.autocall_save = self.shell.autocall
470 470 self.shell.autocall = 0
471 471 else:
472 472 try:
473 473 self.shell.autocall = self._magic_state.autocall_save
474 474 except AttributeError:
475 475 self.shell.autocall = self._magic_state.autocall_save = 1
476 476
477 477 print "Automatic calling is:",['OFF','Smart','Full'][self.shell.autocall]
478 478
479 479
480 480 def magic_page(self, parameter_s=''):
481 481 """Pretty print the object and display it through a pager.
482 482
483 483 %page [options] OBJECT
484 484
485 485 If no object is given, use _ (last output).
486 486
487 487 Options:
488 488
489 489 -r: page str(object), don't pretty-print it."""
490 490
491 491 # After a function contributed by Olivier Aubert, slightly modified.
492 492
493 493 # Process options/args
494 494 opts,args = self.parse_options(parameter_s,'r')
495 495 raw = 'r' in opts
496 496
497 497 oname = args and args or '_'
498 498 info = self._ofind(oname)
499 499 if info['found']:
500 500 txt = (raw and str or pformat)( info['obj'] )
501 501 page.page(txt)
502 502 else:
503 503 print 'Object `%s` not found' % oname
504 504
505 505 def magic_profile(self, parameter_s=''):
506 506 """Print your currently active IPython profile."""
507 507 from IPython.core.application import BaseIPythonApplication
508 508 if BaseIPythonApplication.initialized():
509 509 print BaseIPythonApplication.instance().profile
510 510 else:
511 511 error("profile is an application-level value, but you don't appear to be in an IPython application")
512 512
513 513 def magic_pinfo(self, parameter_s='', namespaces=None):
514 514 """Provide detailed information about an object.
515 515
516 516 '%pinfo object' is just a synonym for object? or ?object."""
517 517
518 518 #print 'pinfo par: <%s>' % parameter_s # dbg
519 519
520 520
521 521 # detail_level: 0 -> obj? , 1 -> obj??
522 522 detail_level = 0
523 523 # We need to detect if we got called as 'pinfo pinfo foo', which can
524 524 # happen if the user types 'pinfo foo?' at the cmd line.
525 525 pinfo,qmark1,oname,qmark2 = \
526 526 re.match('(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
527 527 if pinfo or qmark1 or qmark2:
528 528 detail_level = 1
529 529 if "*" in oname:
530 530 self.magic_psearch(oname)
531 531 else:
532 532 self.shell._inspect('pinfo', oname, detail_level=detail_level,
533 533 namespaces=namespaces)
534 534
535 535 def magic_pinfo2(self, parameter_s='', namespaces=None):
536 536 """Provide extra detailed information about an object.
537 537
538 538 '%pinfo2 object' is just a synonym for object?? or ??object."""
539 539 self.shell._inspect('pinfo', parameter_s, detail_level=1,
540 540 namespaces=namespaces)
541 541
542 542 @skip_doctest
543 543 def magic_pdef(self, parameter_s='', namespaces=None):
544 544 """Print the definition header for any callable object.
545 545
546 546 If the object is a class, print the constructor information.
547 547
548 548 Examples
549 549 --------
550 550 ::
551 551
552 552 In [3]: %pdef urllib.urlopen
553 553 urllib.urlopen(url, data=None, proxies=None)
554 554 """
555 555 self._inspect('pdef',parameter_s, namespaces)
556 556
557 557 def magic_pdoc(self, parameter_s='', namespaces=None):
558 558 """Print the docstring for an object.
559 559
560 560 If the given object is a class, it will print both the class and the
561 561 constructor docstrings."""
562 562 self._inspect('pdoc',parameter_s, namespaces)
563 563
564 564 def magic_psource(self, parameter_s='', namespaces=None):
565 565 """Print (or run through pager) the source code for an object."""
566 566 self._inspect('psource',parameter_s, namespaces)
567 567
568 568 def magic_pfile(self, parameter_s=''):
569 569 """Print (or run through pager) the file where an object is defined.
570 570
571 571 The file opens at the line where the object definition begins. IPython
572 572 will honor the environment variable PAGER if set, and otherwise will
573 573 do its best to print the file in a convenient form.
574 574
575 575 If the given argument is not an object currently defined, IPython will
576 576 try to interpret it as a filename (automatically adding a .py extension
577 577 if needed). You can thus use %pfile as a syntax highlighting code
578 578 viewer."""
579 579
580 580 # first interpret argument as an object name
581 581 out = self._inspect('pfile',parameter_s)
582 582 # if not, try the input as a filename
583 583 if out == 'not found':
584 584 try:
585 585 filename = get_py_filename(parameter_s)
586 586 except IOError,msg:
587 587 print msg
588 588 return
589 589 page.page(self.shell.inspector.format(open(filename).read()))
590 590
591 591 def magic_psearch(self, parameter_s=''):
592 592 """Search for object in namespaces by wildcard.
593 593
594 594 %psearch [options] PATTERN [OBJECT TYPE]
595 595
596 596 Note: ? can be used as a synonym for %psearch, at the beginning or at
597 597 the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the
598 598 rest of the command line must be unchanged (options come first), so
599 599 for example the following forms are equivalent
600 600
601 601 %psearch -i a* function
602 602 -i a* function?
603 603 ?-i a* function
604 604
605 605 Arguments:
606 606
607 607 PATTERN
608 608
609 609 where PATTERN is a string containing * as a wildcard similar to its
610 610 use in a shell. The pattern is matched in all namespaces on the
611 611 search path. By default objects starting with a single _ are not
612 612 matched, many IPython generated objects have a single
613 613 underscore. The default is case insensitive matching. Matching is
614 614 also done on the attributes of objects and not only on the objects
615 615 in a module.
616 616
617 617 [OBJECT TYPE]
618 618
619 619 Is the name of a python type from the types module. The name is
620 620 given in lowercase without the ending type, ex. StringType is
621 621 written string. By adding a type here only objects matching the
622 622 given type are matched. Using all here makes the pattern match all
623 623 types (this is the default).
624 624
625 625 Options:
626 626
627 627 -a: makes the pattern match even objects whose names start with a
628 628 single underscore. These names are normally omitted from the
629 629 search.
630 630
631 631 -i/-c: make the pattern case insensitive/sensitive. If neither of
632 632 these options are given, the default is read from your configuration
633 633 file, with the option ``InteractiveShell.wildcards_case_sensitive``.
634 634 If this option is not specified in your configuration file, IPython's
635 635 internal default is to do a case sensitive search.
636 636
637 637 -e/-s NAMESPACE: exclude/search a given namespace. The pattern you
638 638 specify can be searched in any of the following namespaces:
639 639 'builtin', 'user', 'user_global','internal', 'alias', where
640 640 'builtin' and 'user' are the search defaults. Note that you should
641 641 not use quotes when specifying namespaces.
642 642
643 643 'Builtin' contains the python module builtin, 'user' contains all
644 644 user data, 'alias' only contain the shell aliases and no python
645 645 objects, 'internal' contains objects used by IPython. The
646 646 'user_global' namespace is only used by embedded IPython instances,
647 647 and it contains module-level globals. You can add namespaces to the
648 648 search with -s or exclude them with -e (these options can be given
649 649 more than once).
650 650
651 651 Examples
652 652 --------
653 653 ::
654 654
655 655 %psearch a* -> objects beginning with an a
656 656 %psearch -e builtin a* -> objects NOT in the builtin space starting in a
657 657 %psearch a* function -> all functions beginning with an a
658 658 %psearch re.e* -> objects beginning with an e in module re
659 659 %psearch r*.e* -> objects that start with e in modules starting in r
660 660 %psearch r*.* string -> all strings in modules beginning with r
661 661
662 662 Case sensitive search::
663 663
664 664 %psearch -c a* list all object beginning with lower case a
665 665
666 666 Show objects beginning with a single _::
667 667
668 668 %psearch -a _* list objects beginning with a single underscore"""
669 669 try:
670 670 parameter_s.encode('ascii')
671 671 except UnicodeEncodeError:
672 672 print 'Python identifiers can only contain ascii characters.'
673 673 return
674 674
675 675 # default namespaces to be searched
676 676 def_search = ['user_local', 'user_global', 'builtin']
677 677
678 678 # Process options/args
679 679 opts,args = self.parse_options(parameter_s,'cias:e:',list_all=True)
680 680 opt = opts.get
681 681 shell = self.shell
682 682 psearch = shell.inspector.psearch
683 683
684 684 # select case options
685 685 if opts.has_key('i'):
686 686 ignore_case = True
687 687 elif opts.has_key('c'):
688 688 ignore_case = False
689 689 else:
690 690 ignore_case = not shell.wildcards_case_sensitive
691 691
692 692 # Build list of namespaces to search from user options
693 693 def_search.extend(opt('s',[]))
694 694 ns_exclude = ns_exclude=opt('e',[])
695 695 ns_search = [nm for nm in def_search if nm not in ns_exclude]
696 696
697 697 # Call the actual search
698 698 try:
699 699 psearch(args,shell.ns_table,ns_search,
700 700 show_all=opt('a'),ignore_case=ignore_case)
701 701 except:
702 702 shell.showtraceback()
703 703
704 704 @skip_doctest
705 705 def magic_who_ls(self, parameter_s=''):
706 706 """Return a sorted list of all interactive variables.
707 707
708 708 If arguments are given, only variables of types matching these
709 709 arguments are returned.
710 710
711 711 Examples
712 712 --------
713 713
714 714 Define two variables and list them with who_ls::
715 715
716 716 In [1]: alpha = 123
717 717
718 718 In [2]: beta = 'test'
719 719
720 720 In [3]: %who_ls
721 721 Out[3]: ['alpha', 'beta']
722 722
723 723 In [4]: %who_ls int
724 724 Out[4]: ['alpha']
725 725
726 726 In [5]: %who_ls str
727 727 Out[5]: ['beta']
728 728 """
729 729
730 730 user_ns = self.shell.user_ns
731 731 user_ns_hidden = self.shell.user_ns_hidden
732 732 out = [ i for i in user_ns
733 733 if not i.startswith('_') \
734 734 and not i in user_ns_hidden ]
735 735
736 736 typelist = parameter_s.split()
737 737 if typelist:
738 738 typeset = set(typelist)
739 739 out = [i for i in out if type(user_ns[i]).__name__ in typeset]
740 740
741 741 out.sort()
742 742 return out
743 743
744 744 @skip_doctest
745 745 def magic_who(self, parameter_s=''):
746 746 """Print all interactive variables, with some minimal formatting.
747 747
748 748 If any arguments are given, only variables whose type matches one of
749 749 these are printed. For example::
750 750
751 751 %who function str
752 752
753 753 will only list functions and strings, excluding all other types of
754 754 variables. To find the proper type names, simply use type(var) at a
755 755 command line to see how python prints type names. For example:
756 756
757 757 ::
758 758
759 759 In [1]: type('hello')\\
760 760 Out[1]: <type 'str'>
761 761
762 762 indicates that the type name for strings is 'str'.
763 763
764 764 ``%who`` always excludes executed names loaded through your configuration
765 765 file and things which are internal to IPython.
766 766
767 767 This is deliberate, as typically you may load many modules and the
768 768 purpose of %who is to show you only what you've manually defined.
769 769
770 770 Examples
771 771 --------
772 772
773 773 Define two variables and list them with who::
774 774
775 775 In [1]: alpha = 123
776 776
777 777 In [2]: beta = 'test'
778 778
779 779 In [3]: %who
780 780 alpha beta
781 781
782 782 In [4]: %who int
783 783 alpha
784 784
785 785 In [5]: %who str
786 786 beta
787 787 """
788 788
789 789 varlist = self.magic_who_ls(parameter_s)
790 790 if not varlist:
791 791 if parameter_s:
792 792 print 'No variables match your requested type.'
793 793 else:
794 794 print 'Interactive namespace is empty.'
795 795 return
796 796
797 797 # if we have variables, move on...
798 798 count = 0
799 799 for i in varlist:
800 800 print i+'\t',
801 801 count += 1
802 802 if count > 8:
803 803 count = 0
804 804 print
805 805 print
806 806
807 807 @skip_doctest
808 808 def magic_whos(self, parameter_s=''):
809 809 """Like %who, but gives some extra information about each variable.
810 810
811 811 The same type filtering of %who can be applied here.
812 812
813 813 For all variables, the type is printed. Additionally it prints:
814 814
815 815 - For {},[],(): their length.
816 816
817 817 - For numpy arrays, a summary with shape, number of
818 818 elements, typecode and size in memory.
819 819
820 820 - Everything else: a string representation, snipping their middle if
821 821 too long.
822 822
823 823 Examples
824 824 --------
825 825
826 826 Define two variables and list them with whos::
827 827
828 828 In [1]: alpha = 123
829 829
830 830 In [2]: beta = 'test'
831 831
832 832 In [3]: %whos
833 833 Variable Type Data/Info
834 834 --------------------------------
835 835 alpha int 123
836 836 beta str test
837 837 """
838 838
839 839 varnames = self.magic_who_ls(parameter_s)
840 840 if not varnames:
841 841 if parameter_s:
842 842 print 'No variables match your requested type.'
843 843 else:
844 844 print 'Interactive namespace is empty.'
845 845 return
846 846
847 847 # if we have variables, move on...
848 848
849 849 # for these types, show len() instead of data:
850 850 seq_types = ['dict', 'list', 'tuple']
851 851
852 852 # for numpy arrays, display summary info
853 853 ndarray_type = None
854 854 if 'numpy' in sys.modules:
855 855 try:
856 856 from numpy import ndarray
857 857 except ImportError:
858 858 pass
859 859 else:
860 860 ndarray_type = ndarray.__name__
861 861
862 862 # Find all variable names and types so we can figure out column sizes
863 863 def get_vars(i):
864 864 return self.shell.user_ns[i]
865 865
866 866 # some types are well known and can be shorter
867 867 abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
868 868 def type_name(v):
869 869 tn = type(v).__name__
870 870 return abbrevs.get(tn,tn)
871 871
872 872 varlist = map(get_vars,varnames)
873 873
874 874 typelist = []
875 875 for vv in varlist:
876 876 tt = type_name(vv)
877 877
878 878 if tt=='instance':
879 879 typelist.append( abbrevs.get(str(vv.__class__),
880 880 str(vv.__class__)))
881 881 else:
882 882 typelist.append(tt)
883 883
884 884 # column labels and # of spaces as separator
885 885 varlabel = 'Variable'
886 886 typelabel = 'Type'
887 887 datalabel = 'Data/Info'
888 888 colsep = 3
889 889 # variable format strings
890 890 vformat = "{0:<{varwidth}}{1:<{typewidth}}"
891 891 aformat = "%s: %s elems, type `%s`, %s bytes"
892 892 # find the size of the columns to format the output nicely
893 893 varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
894 894 typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
895 895 # table header
896 896 print varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
897 897 ' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1)
898 898 # and the table itself
899 899 kb = 1024
900 900 Mb = 1048576 # kb**2
901 901 for vname,var,vtype in zip(varnames,varlist,typelist):
902 902 print vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth),
903 903 if vtype in seq_types:
904 904 print "n="+str(len(var))
905 905 elif vtype == ndarray_type:
906 906 vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
907 907 if vtype==ndarray_type:
908 908 # numpy
909 909 vsize = var.size
910 910 vbytes = vsize*var.itemsize
911 911 vdtype = var.dtype
912 912
913 913 if vbytes < 100000:
914 914 print aformat % (vshape,vsize,vdtype,vbytes)
915 915 else:
916 916 print aformat % (vshape,vsize,vdtype,vbytes),
917 917 if vbytes < Mb:
918 918 print '(%s kb)' % (vbytes/kb,)
919 919 else:
920 920 print '(%s Mb)' % (vbytes/Mb,)
921 921 else:
922 922 try:
923 923 vstr = str(var)
924 924 except UnicodeEncodeError:
925 925 vstr = unicode(var).encode(DEFAULT_ENCODING,
926 926 'backslashreplace')
927 927 except:
928 928 vstr = "<object with id %d (str() failed)>" % id(var)
929 929 vstr = vstr.replace('\n','\\n')
930 930 if len(vstr) < 50:
931 931 print vstr
932 932 else:
933 933 print vstr[:25] + "<...>" + vstr[-25:]
934 934
935 935 def magic_reset(self, parameter_s=''):
936 936 """Resets the namespace by removing all names defined by the user, if
937 937 called without arguments, or by removing some types of objects, such
938 938 as everything currently in IPython's In[] and Out[] containers (see
939 939 the parameters for details).
940 940
941 941 Parameters
942 942 ----------
943 943 -f : force reset without asking for confirmation.
944 944
945 945 -s : 'Soft' reset: Only clears your namespace, leaving history intact.
946 946 References to objects may be kept. By default (without this option),
947 947 we do a 'hard' reset, giving you a new session and removing all
948 948 references to objects from the current session.
949 949
950 950 in : reset input history
951 951
952 952 out : reset output history
953 953
954 954 dhist : reset directory history
955 955
956 956 array : reset only variables that are NumPy arrays
957 957
958 958 See Also
959 959 --------
960 960 magic_reset_selective : invoked as ``%reset_selective``
961 961
962 962 Examples
963 963 --------
964 964 ::
965 965
966 966 In [6]: a = 1
967 967
968 968 In [7]: a
969 969 Out[7]: 1
970 970
971 971 In [8]: 'a' in _ip.user_ns
972 972 Out[8]: True
973 973
974 974 In [9]: %reset -f
975 975
976 976 In [1]: 'a' in _ip.user_ns
977 977 Out[1]: False
978 978
979 979 In [2]: %reset -f in
980 980 Flushing input history
981 981
982 982 In [3]: %reset -f dhist in
983 983 Flushing directory history
984 984 Flushing input history
985 985
986 986 Notes
987 987 -----
988 988 Calling this magic from clients that do not implement standard input,
989 989 such as the ipython notebook interface, will reset the namespace
990 990 without confirmation.
991 991 """
992 992 opts, args = self.parse_options(parameter_s,'sf', mode='list')
993 993 if 'f' in opts:
994 994 ans = True
995 995 else:
996 996 try:
997 997 ans = self.shell.ask_yes_no(
998 998 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ", default='n')
999 999 except StdinNotImplementedError:
1000 1000 ans = True
1001 1001 if not ans:
1002 1002 print 'Nothing done.'
1003 1003 return
1004 1004
1005 1005 if 's' in opts: # Soft reset
1006 1006 user_ns = self.shell.user_ns
1007 1007 for i in self.magic_who_ls():
1008 1008 del(user_ns[i])
1009 1009 elif len(args) == 0: # Hard reset
1010 1010 self.shell.reset(new_session = False)
1011 1011
1012 1012 # reset in/out/dhist/array: previously extensinions/clearcmd.py
1013 1013 ip = self.shell
1014 1014 user_ns = self.shell.user_ns # local lookup, heavily used
1015 1015
1016 1016 for target in args:
1017 1017 target = target.lower() # make matches case insensitive
1018 1018 if target == 'out':
1019 1019 print "Flushing output cache (%d entries)" % len(user_ns['_oh'])
1020 1020 self.shell.displayhook.flush()
1021 1021
1022 1022 elif target == 'in':
1023 1023 print "Flushing input history"
1024 1024 pc = self.shell.displayhook.prompt_count + 1
1025 1025 for n in range(1, pc):
1026 1026 key = '_i'+repr(n)
1027 1027 user_ns.pop(key,None)
1028 1028 user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
1029 1029 hm = ip.history_manager
1030 1030 # don't delete these, as %save and %macro depending on the length
1031 1031 # of these lists to be preserved
1032 1032 hm.input_hist_parsed[:] = [''] * pc
1033 1033 hm.input_hist_raw[:] = [''] * pc
1034 1034 # hm has internal machinery for _i,_ii,_iii, clear it out
1035 1035 hm._i = hm._ii = hm._iii = hm._i00 = u''
1036 1036
1037 1037 elif target == 'array':
1038 1038 # Support cleaning up numpy arrays
1039 1039 try:
1040 1040 from numpy import ndarray
1041 1041 # This must be done with items and not iteritems because we're
1042 1042 # going to modify the dict in-place.
1043 1043 for x,val in user_ns.items():
1044 1044 if isinstance(val,ndarray):
1045 1045 del user_ns[x]
1046 1046 except ImportError:
1047 1047 print "reset array only works if Numpy is available."
1048 1048
1049 1049 elif target == 'dhist':
1050 1050 print "Flushing directory history"
1051 1051 del user_ns['_dh'][:]
1052 1052
1053 1053 else:
1054 1054 print "Don't know how to reset ",
1055 1055 print target + ", please run `%reset?` for details"
1056 1056
1057 1057 gc.collect()
1058 1058
1059 1059 def magic_reset_selective(self, parameter_s=''):
1060 1060 """Resets the namespace by removing names defined by the user.
1061 1061
1062 1062 Input/Output history are left around in case you need them.
1063 1063
1064 1064 %reset_selective [-f] regex
1065 1065
1066 1066 No action is taken if regex is not included
1067 1067
1068 1068 Options
1069 1069 -f : force reset without asking for confirmation.
1070 1070
1071 1071 See Also
1072 1072 --------
1073 1073 magic_reset : invoked as ``%reset``
1074 1074
1075 1075 Examples
1076 1076 --------
1077 1077
1078 1078 We first fully reset the namespace so your output looks identical to
1079 1079 this example for pedagogical reasons; in practice you do not need a
1080 1080 full reset::
1081 1081
1082 1082 In [1]: %reset -f
1083 1083
1084 1084 Now, with a clean namespace we can make a few variables and use
1085 1085 ``%reset_selective`` to only delete names that match our regexp::
1086 1086
1087 1087 In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
1088 1088
1089 1089 In [3]: who_ls
1090 1090 Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
1091 1091
1092 1092 In [4]: %reset_selective -f b[2-3]m
1093 1093
1094 1094 In [5]: who_ls
1095 1095 Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
1096 1096
1097 1097 In [6]: %reset_selective -f d
1098 1098
1099 1099 In [7]: who_ls
1100 1100 Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
1101 1101
1102 1102 In [8]: %reset_selective -f c
1103 1103
1104 1104 In [9]: who_ls
1105 1105 Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
1106 1106
1107 1107 In [10]: %reset_selective -f b
1108 1108
1109 1109 In [11]: who_ls
1110 1110 Out[11]: ['a']
1111 1111
1112 1112 Notes
1113 1113 -----
1114 1114 Calling this magic from clients that do not implement standard input,
1115 1115 such as the ipython notebook interface, will reset the namespace
1116 1116 without confirmation.
1117 1117 """
1118 1118
1119 1119 opts, regex = self.parse_options(parameter_s,'f')
1120 1120
1121 1121 if opts.has_key('f'):
1122 1122 ans = True
1123 1123 else:
1124 1124 try:
1125 1125 ans = self.shell.ask_yes_no(
1126 1126 "Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
1127 1127 default='n')
1128 1128 except StdinNotImplementedError:
1129 1129 ans = True
1130 1130 if not ans:
1131 1131 print 'Nothing done.'
1132 1132 return
1133 1133 user_ns = self.shell.user_ns
1134 1134 if not regex:
1135 1135 print 'No regex pattern specified. Nothing done.'
1136 1136 return
1137 1137 else:
1138 1138 try:
1139 1139 m = re.compile(regex)
1140 1140 except TypeError:
1141 1141 raise TypeError('regex must be a string or compiled pattern')
1142 1142 for i in self.magic_who_ls():
1143 1143 if m.search(i):
1144 1144 del(user_ns[i])
1145 1145
1146 1146 def magic_xdel(self, parameter_s=''):
1147 1147 """Delete a variable, trying to clear it from anywhere that
1148 1148 IPython's machinery has references to it. By default, this uses
1149 1149 the identity of the named object in the user namespace to remove
1150 1150 references held under other names. The object is also removed
1151 1151 from the output history.
1152 1152
1153 1153 Options
1154 1154 -n : Delete the specified name from all namespaces, without
1155 1155 checking their identity.
1156 1156 """
1157 1157 opts, varname = self.parse_options(parameter_s,'n')
1158 1158 try:
1159 1159 self.shell.del_var(varname, ('n' in opts))
1160 1160 except (NameError, ValueError) as e:
1161 1161 print type(e).__name__ +": "+ str(e)
1162 1162
1163 1163 def magic_logstart(self,parameter_s=''):
1164 1164 """Start logging anywhere in a session.
1165 1165
1166 1166 %logstart [-o|-r|-t] [log_name [log_mode]]
1167 1167
1168 1168 If no name is given, it defaults to a file named 'ipython_log.py' in your
1169 1169 current directory, in 'rotate' mode (see below).
1170 1170
1171 1171 '%logstart name' saves to file 'name' in 'backup' mode. It saves your
1172 1172 history up to that point and then continues logging.
1173 1173
1174 1174 %logstart takes a second optional parameter: logging mode. This can be one
1175 1175 of (note that the modes are given unquoted):\\
1176 1176 append: well, that says it.\\
1177 1177 backup: rename (if exists) to name~ and start name.\\
1178 1178 global: single logfile in your home dir, appended to.\\
1179 1179 over : overwrite existing log.\\
1180 1180 rotate: create rotating logs name.1~, name.2~, etc.
1181 1181
1182 1182 Options:
1183 1183
1184 1184 -o: log also IPython's output. In this mode, all commands which
1185 1185 generate an Out[NN] prompt are recorded to the logfile, right after
1186 1186 their corresponding input line. The output lines are always
1187 1187 prepended with a '#[Out]# ' marker, so that the log remains valid
1188 1188 Python code.
1189 1189
1190 1190 Since this marker is always the same, filtering only the output from
1191 1191 a log is very easy, using for example a simple awk call::
1192 1192
1193 1193 awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
1194 1194
1195 1195 -r: log 'raw' input. Normally, IPython's logs contain the processed
1196 1196 input, so that user lines are logged in their final form, converted
1197 1197 into valid Python. For example, %Exit is logged as
1198 1198 _ip.magic("Exit"). If the -r flag is given, all input is logged
1199 1199 exactly as typed, with no transformations applied.
1200 1200
1201 1201 -t: put timestamps before each input line logged (these are put in
1202 1202 comments)."""
1203 1203
1204 1204 opts,par = self.parse_options(parameter_s,'ort')
1205 1205 log_output = 'o' in opts
1206 1206 log_raw_input = 'r' in opts
1207 1207 timestamp = 't' in opts
1208 1208
1209 1209 logger = self.shell.logger
1210 1210
1211 1211 # if no args are given, the defaults set in the logger constructor by
1212 1212 # ipython remain valid
1213 1213 if par:
1214 1214 try:
1215 1215 logfname,logmode = par.split()
1216 1216 except:
1217 1217 logfname = par
1218 1218 logmode = 'backup'
1219 1219 else:
1220 1220 logfname = logger.logfname
1221 1221 logmode = logger.logmode
1222 1222 # put logfname into rc struct as if it had been called on the command
1223 1223 # line, so it ends up saved in the log header Save it in case we need
1224 1224 # to restore it...
1225 1225 old_logfile = self.shell.logfile
1226 1226 if logfname:
1227 1227 logfname = os.path.expanduser(logfname)
1228 1228 self.shell.logfile = logfname
1229 1229
1230 1230 loghead = '# IPython log file\n\n'
1231 1231 try:
1232 1232 started = logger.logstart(logfname,loghead,logmode,
1233 1233 log_output,timestamp,log_raw_input)
1234 1234 except:
1235 1235 self.shell.logfile = old_logfile
1236 1236 warn("Couldn't start log: %s" % sys.exc_info()[1])
1237 1237 else:
1238 1238 # log input history up to this point, optionally interleaving
1239 1239 # output if requested
1240 1240
1241 1241 if timestamp:
1242 1242 # disable timestamping for the previous history, since we've
1243 1243 # lost those already (no time machine here).
1244 1244 logger.timestamp = False
1245 1245
1246 1246 if log_raw_input:
1247 1247 input_hist = self.shell.history_manager.input_hist_raw
1248 1248 else:
1249 1249 input_hist = self.shell.history_manager.input_hist_parsed
1250 1250
1251 1251 if log_output:
1252 1252 log_write = logger.log_write
1253 1253 output_hist = self.shell.history_manager.output_hist
1254 1254 for n in range(1,len(input_hist)-1):
1255 1255 log_write(input_hist[n].rstrip() + '\n')
1256 1256 if n in output_hist:
1257 1257 log_write(repr(output_hist[n]),'output')
1258 1258 else:
1259 1259 logger.log_write('\n'.join(input_hist[1:]))
1260 1260 logger.log_write('\n')
1261 1261 if timestamp:
1262 1262 # re-enable timestamping
1263 1263 logger.timestamp = True
1264 1264
1265 1265 print ('Activating auto-logging. '
1266 1266 'Current session state plus future input saved.')
1267 1267 logger.logstate()
1268 1268
1269 1269 def magic_logstop(self,parameter_s=''):
1270 1270 """Fully stop logging and close log file.
1271 1271
1272 1272 In order to start logging again, a new %logstart call needs to be made,
1273 1273 possibly (though not necessarily) with a new filename, mode and other
1274 1274 options."""
1275 1275 self.logger.logstop()
1276 1276
1277 1277 def magic_logoff(self,parameter_s=''):
1278 1278 """Temporarily stop logging.
1279 1279
1280 1280 You must have previously started logging."""
1281 1281 self.shell.logger.switch_log(0)
1282 1282
1283 1283 def magic_logon(self,parameter_s=''):
1284 1284 """Restart logging.
1285 1285
1286 1286 This function is for restarting logging which you've temporarily
1287 1287 stopped with %logoff. For starting logging for the first time, you
1288 1288 must use the %logstart function, which allows you to specify an
1289 1289 optional log filename."""
1290 1290
1291 1291 self.shell.logger.switch_log(1)
1292 1292
1293 1293 def magic_logstate(self,parameter_s=''):
1294 1294 """Print the status of the logging system."""
1295 1295
1296 1296 self.shell.logger.logstate()
1297 1297
1298 1298 def magic_pdb(self, parameter_s=''):
1299 1299 """Control the automatic calling of the pdb interactive debugger.
1300 1300
1301 1301 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
1302 1302 argument it works as a toggle.
1303 1303
1304 1304 When an exception is triggered, IPython can optionally call the
1305 1305 interactive pdb debugger after the traceback printout. %pdb toggles
1306 1306 this feature on and off.
1307 1307
1308 1308 The initial state of this feature is set in your configuration
1309 1309 file (the option is ``InteractiveShell.pdb``).
1310 1310
1311 1311 If you want to just activate the debugger AFTER an exception has fired,
1312 1312 without having to type '%pdb on' and rerunning your code, you can use
1313 1313 the %debug magic."""
1314 1314
1315 1315 par = parameter_s.strip().lower()
1316 1316
1317 1317 if par:
1318 1318 try:
1319 1319 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
1320 1320 except KeyError:
1321 1321 print ('Incorrect argument. Use on/1, off/0, '
1322 1322 'or nothing for a toggle.')
1323 1323 return
1324 1324 else:
1325 1325 # toggle
1326 1326 new_pdb = not self.shell.call_pdb
1327 1327
1328 1328 # set on the shell
1329 1329 self.shell.call_pdb = new_pdb
1330 1330 print 'Automatic pdb calling has been turned',on_off(new_pdb)
1331 1331
1332 1332 def magic_debug(self, parameter_s=''):
1333 1333 """Activate the interactive debugger in post-mortem mode.
1334 1334
1335 1335 If an exception has just occurred, this lets you inspect its stack
1336 1336 frames interactively. Note that this will always work only on the last
1337 1337 traceback that occurred, so you must call this quickly after an
1338 1338 exception that you wish to inspect has fired, because if another one
1339 1339 occurs, it clobbers the previous one.
1340 1340
1341 1341 If you want IPython to automatically do this on every exception, see
1342 1342 the %pdb magic for more details.
1343 1343 """
1344 1344 self.shell.debugger(force=True)
1345 1345
1346 1346 @skip_doctest
1347 1347 def magic_prun(self, parameter_s ='',user_mode=1,
1348 1348 opts=None,arg_lst=None,prog_ns=None):
1349 1349
1350 1350 """Run a statement through the python code profiler.
1351 1351
1352 1352 Usage:
1353 1353 %prun [options] statement
1354 1354
1355 1355 The given statement (which doesn't require quote marks) is run via the
1356 1356 python profiler in a manner similar to the profile.run() function.
1357 1357 Namespaces are internally managed to work correctly; profile.run
1358 1358 cannot be used in IPython because it makes certain assumptions about
1359 1359 namespaces which do not hold under IPython.
1360 1360
1361 1361 Options:
1362 1362
1363 1363 -l <limit>: you can place restrictions on what or how much of the
1364 1364 profile gets printed. The limit value can be:
1365 1365
1366 1366 * A string: only information for function names containing this string
1367 1367 is printed.
1368 1368
1369 1369 * An integer: only these many lines are printed.
1370 1370
1371 1371 * A float (between 0 and 1): this fraction of the report is printed
1372 1372 (for example, use a limit of 0.4 to see the topmost 40% only).
1373 1373
1374 1374 You can combine several limits with repeated use of the option. For
1375 1375 example, '-l __init__ -l 5' will print only the topmost 5 lines of
1376 1376 information about class constructors.
1377 1377
1378 1378 -r: return the pstats.Stats object generated by the profiling. This
1379 1379 object has all the information about the profile in it, and you can
1380 1380 later use it for further analysis or in other functions.
1381 1381
1382 1382 -s <key>: sort profile by given key. You can provide more than one key
1383 1383 by using the option several times: '-s key1 -s key2 -s key3...'. The
1384 1384 default sorting key is 'time'.
1385 1385
1386 1386 The following is copied verbatim from the profile documentation
1387 1387 referenced below:
1388 1388
1389 1389 When more than one key is provided, additional keys are used as
1390 1390 secondary criteria when the there is equality in all keys selected
1391 1391 before them.
1392 1392
1393 1393 Abbreviations can be used for any key names, as long as the
1394 1394 abbreviation is unambiguous. The following are the keys currently
1395 1395 defined:
1396 1396
1397 1397 Valid Arg Meaning
1398 1398 "calls" call count
1399 1399 "cumulative" cumulative time
1400 1400 "file" file name
1401 1401 "module" file name
1402 1402 "pcalls" primitive call count
1403 1403 "line" line number
1404 1404 "name" function name
1405 1405 "nfl" name/file/line
1406 1406 "stdname" standard name
1407 1407 "time" internal time
1408 1408
1409 1409 Note that all sorts on statistics are in descending order (placing
1410 1410 most time consuming items first), where as name, file, and line number
1411 1411 searches are in ascending order (i.e., alphabetical). The subtle
1412 1412 distinction between "nfl" and "stdname" is that the standard name is a
1413 1413 sort of the name as printed, which means that the embedded line
1414 1414 numbers get compared in an odd way. For example, lines 3, 20, and 40
1415 1415 would (if the file names were the same) appear in the string order
1416 1416 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
1417 1417 line numbers. In fact, sort_stats("nfl") is the same as
1418 1418 sort_stats("name", "file", "line").
1419 1419
1420 1420 -T <filename>: save profile results as shown on screen to a text
1421 1421 file. The profile is still shown on screen.
1422 1422
1423 1423 -D <filename>: save (via dump_stats) profile statistics to given
1424 1424 filename. This data is in a format understood by the pstats module, and
1425 1425 is generated by a call to the dump_stats() method of profile
1426 1426 objects. The profile is still shown on screen.
1427 1427
1428 1428 -q: suppress output to the pager. Best used with -T and/or -D above.
1429 1429
1430 1430 If you want to run complete programs under the profiler's control, use
1431 1431 '%run -p [prof_opts] filename.py [args to program]' where prof_opts
1432 1432 contains profiler specific options as described here.
1433 1433
1434 1434 You can read the complete documentation for the profile module with::
1435 1435
1436 1436 In [1]: import profile; profile.help()
1437 1437 """
1438 1438
1439 1439 opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
1440 1440
1441 1441 if user_mode: # regular user call
1442 1442 opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:q',
1443 1443 list_all=1, posix=False)
1444 1444 namespace = self.shell.user_ns
1445 1445 else: # called to run a program by %run -p
1446 1446 try:
1447 1447 filename = get_py_filename(arg_lst[0])
1448 1448 except IOError as e:
1449 1449 try:
1450 1450 msg = str(e)
1451 1451 except UnicodeError:
1452 1452 msg = e.message
1453 1453 error(msg)
1454 1454 return
1455 1455
1456 1456 arg_str = 'execfile(filename,prog_ns)'
1457 1457 namespace = {
1458 1458 'execfile': self.shell.safe_execfile,
1459 1459 'prog_ns': prog_ns,
1460 1460 'filename': filename
1461 1461 }
1462 1462
1463 1463 opts.merge(opts_def)
1464 1464
1465 1465 prof = profile.Profile()
1466 1466 try:
1467 1467 prof = prof.runctx(arg_str,namespace,namespace)
1468 1468 sys_exit = ''
1469 1469 except SystemExit:
1470 1470 sys_exit = """*** SystemExit exception caught in code being profiled."""
1471 1471
1472 1472 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
1473 1473
1474 1474 lims = opts.l
1475 1475 if lims:
1476 1476 lims = [] # rebuild lims with ints/floats/strings
1477 1477 for lim in opts.l:
1478 1478 try:
1479 1479 lims.append(int(lim))
1480 1480 except ValueError:
1481 1481 try:
1482 1482 lims.append(float(lim))
1483 1483 except ValueError:
1484 1484 lims.append(lim)
1485 1485
1486 1486 # Trap output.
1487 1487 stdout_trap = StringIO()
1488 1488
1489 1489 if hasattr(stats,'stream'):
1490 1490 # In newer versions of python, the stats object has a 'stream'
1491 1491 # attribute to write into.
1492 1492 stats.stream = stdout_trap
1493 1493 stats.print_stats(*lims)
1494 1494 else:
1495 1495 # For older versions, we manually redirect stdout during printing
1496 1496 sys_stdout = sys.stdout
1497 1497 try:
1498 1498 sys.stdout = stdout_trap
1499 1499 stats.print_stats(*lims)
1500 1500 finally:
1501 1501 sys.stdout = sys_stdout
1502 1502
1503 1503 output = stdout_trap.getvalue()
1504 1504 output = output.rstrip()
1505 1505
1506 1506 if 'q' not in opts:
1507 1507 page.page(output)
1508 1508 print sys_exit,
1509 1509
1510 1510 dump_file = opts.D[0]
1511 1511 text_file = opts.T[0]
1512 1512 if dump_file:
1513 1513 dump_file = unquote_filename(dump_file)
1514 1514 prof.dump_stats(dump_file)
1515 1515 print '\n*** Profile stats marshalled to file',\
1516 1516 `dump_file`+'.',sys_exit
1517 1517 if text_file:
1518 1518 text_file = unquote_filename(text_file)
1519 1519 pfile = open(text_file,'w')
1520 1520 pfile.write(output)
1521 1521 pfile.close()
1522 1522 print '\n*** Profile printout saved to text file',\
1523 1523 `text_file`+'.',sys_exit
1524 1524
1525 1525 if opts.has_key('r'):
1526 1526 return stats
1527 1527 else:
1528 1528 return None
1529 1529
1530 1530 @skip_doctest
1531 1531 def magic_run(self, parameter_s ='', runner=None,
1532 1532 file_finder=get_py_filename):
1533 1533 """Run the named file inside IPython as a program.
1534 1534
1535 1535 Usage:\\
1536 1536 %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] file [args]
1537 1537
1538 1538 Parameters after the filename are passed as command-line arguments to
1539 1539 the program (put in sys.argv). Then, control returns to IPython's
1540 1540 prompt.
1541 1541
1542 1542 This is similar to running at a system prompt:\\
1543 1543 $ python file args\\
1544 1544 but with the advantage of giving you IPython's tracebacks, and of
1545 1545 loading all variables into your interactive namespace for further use
1546 1546 (unless -p is used, see below).
1547 1547
1548 1548 The file is executed in a namespace initially consisting only of
1549 1549 __name__=='__main__' and sys.argv constructed as indicated. It thus
1550 1550 sees its environment as if it were being run as a stand-alone program
1551 1551 (except for sharing global objects such as previously imported
1552 1552 modules). But after execution, the IPython interactive namespace gets
1553 1553 updated with all variables defined in the program (except for __name__
1554 1554 and sys.argv). This allows for very convenient loading of code for
1555 1555 interactive work, while giving each program a 'clean sheet' to run in.
1556 1556
1557 1557 Options:
1558 1558
1559 1559 -n: __name__ is NOT set to '__main__', but to the running file's name
1560 1560 without extension (as python does under import). This allows running
1561 1561 scripts and reloading the definitions in them without calling code
1562 1562 protected by an ' if __name__ == "__main__" ' clause.
1563 1563
1564 1564 -i: run the file in IPython's namespace instead of an empty one. This
1565 1565 is useful if you are experimenting with code written in a text editor
1566 1566 which depends on variables defined interactively.
1567 1567
1568 1568 -e: ignore sys.exit() calls or SystemExit exceptions in the script
1569 1569 being run. This is particularly useful if IPython is being used to
1570 1570 run unittests, which always exit with a sys.exit() call. In such
1571 1571 cases you are interested in the output of the test results, not in
1572 1572 seeing a traceback of the unittest module.
1573 1573
1574 1574 -t: print timing information at the end of the run. IPython will give
1575 1575 you an estimated CPU time consumption for your script, which under
1576 1576 Unix uses the resource module to avoid the wraparound problems of
1577 1577 time.clock(). Under Unix, an estimate of time spent on system tasks
1578 1578 is also given (for Windows platforms this is reported as 0.0).
1579 1579
1580 1580 If -t is given, an additional -N<N> option can be given, where <N>
1581 1581 must be an integer indicating how many times you want the script to
1582 1582 run. The final timing report will include total and per run results.
1583 1583
1584 1584 For example (testing the script uniq_stable.py)::
1585 1585
1586 1586 In [1]: run -t uniq_stable
1587 1587
1588 1588 IPython CPU timings (estimated):\\
1589 1589 User : 0.19597 s.\\
1590 1590 System: 0.0 s.\\
1591 1591
1592 1592 In [2]: run -t -N5 uniq_stable
1593 1593
1594 1594 IPython CPU timings (estimated):\\
1595 1595 Total runs performed: 5\\
1596 1596 Times : Total Per run\\
1597 1597 User : 0.910862 s, 0.1821724 s.\\
1598 1598 System: 0.0 s, 0.0 s.
1599 1599
1600 1600 -d: run your program under the control of pdb, the Python debugger.
1601 1601 This allows you to execute your program step by step, watch variables,
1602 1602 etc. Internally, what IPython does is similar to calling:
1603 1603
1604 1604 pdb.run('execfile("YOURFILENAME")')
1605 1605
1606 1606 with a breakpoint set on line 1 of your file. You can change the line
1607 1607 number for this automatic breakpoint to be <N> by using the -bN option
1608 1608 (where N must be an integer). For example::
1609 1609
1610 1610 %run -d -b40 myscript
1611 1611
1612 1612 will set the first breakpoint at line 40 in myscript.py. Note that
1613 1613 the first breakpoint must be set on a line which actually does
1614 1614 something (not a comment or docstring) for it to stop execution.
1615 1615
1616 1616 When the pdb debugger starts, you will see a (Pdb) prompt. You must
1617 1617 first enter 'c' (without quotes) to start execution up to the first
1618 1618 breakpoint.
1619 1619
1620 1620 Entering 'help' gives information about the use of the debugger. You
1621 1621 can easily see pdb's full documentation with "import pdb;pdb.help()"
1622 1622 at a prompt.
1623 1623
1624 1624 -p: run program under the control of the Python profiler module (which
1625 1625 prints a detailed report of execution times, function calls, etc).
1626 1626
1627 1627 You can pass other options after -p which affect the behavior of the
1628 1628 profiler itself. See the docs for %prun for details.
1629 1629
1630 1630 In this mode, the program's variables do NOT propagate back to the
1631 1631 IPython interactive namespace (because they remain in the namespace
1632 1632 where the profiler executes them).
1633 1633
1634 1634 Internally this triggers a call to %prun, see its documentation for
1635 1635 details on the options available specifically for profiling.
1636 1636
1637 1637 There is one special usage for which the text above doesn't apply:
1638 1638 if the filename ends with .ipy, the file is run as ipython script,
1639 1639 just as if the commands were written on IPython prompt.
1640 1640
1641 1641 -m: specify module name to load instead of script path. Similar to
1642 1642 the -m option for the python interpreter. Use this option last if you
1643 1643 want to combine with other %run options. Unlike the python interpreter
1644 1644 only source modules are allowed no .pyc or .pyo files.
1645 1645 For example::
1646 1646
1647 1647 %run -m example
1648 1648
1649 1649 will run the example module.
1650 1650
1651 1651 """
1652 1652
1653 1653 # get arguments and set sys.argv for program to be run.
1654 1654 opts, arg_lst = self.parse_options(parameter_s, 'nidtN:b:pD:l:rs:T:em:',
1655 1655 mode='list', list_all=1)
1656 1656 if "m" in opts:
1657 1657 modulename = opts["m"][0]
1658 1658 modpath = find_mod(modulename)
1659 1659 if modpath is None:
1660 1660 warn('%r is not a valid modulename on sys.path'%modulename)
1661 1661 return
1662 1662 arg_lst = [modpath] + arg_lst
1663 1663 try:
1664 1664 filename = file_finder(arg_lst[0])
1665 1665 except IndexError:
1666 1666 warn('you must provide at least a filename.')
1667 1667 print '\n%run:\n', oinspect.getdoc(self.magic_run)
1668 1668 return
1669 1669 except IOError as e:
1670 1670 try:
1671 1671 msg = str(e)
1672 1672 except UnicodeError:
1673 1673 msg = e.message
1674 1674 error(msg)
1675 1675 return
1676 1676
1677 1677 if filename.lower().endswith('.ipy'):
1678 1678 self.shell.safe_execfile_ipy(filename)
1679 1679 return
1680 1680
1681 1681 # Control the response to exit() calls made by the script being run
1682 1682 exit_ignore = 'e' in opts
1683 1683
1684 1684 # Make sure that the running script gets a proper sys.argv as if it
1685 1685 # were run from a system shell.
1686 1686 save_argv = sys.argv # save it for later restoring
1687 1687
1688 1688 # simulate shell expansion on arguments, at least tilde expansion
1689 1689 args = [ os.path.expanduser(a) for a in arg_lst[1:] ]
1690 1690
1691 1691 sys.argv = [filename] + args # put in the proper filename
1692 1692 # protect sys.argv from potential unicode strings on Python 2:
1693 1693 if not py3compat.PY3:
1694 1694 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
1695 1695
1696 1696 if 'i' in opts:
1697 1697 # Run in user's interactive namespace
1698 1698 prog_ns = self.shell.user_ns
1699 1699 __name__save = self.shell.user_ns['__name__']
1700 1700 prog_ns['__name__'] = '__main__'
1701 1701 main_mod = self.shell.new_main_mod(prog_ns)
1702 1702 else:
1703 1703 # Run in a fresh, empty namespace
1704 1704 if 'n' in opts:
1705 1705 name = os.path.splitext(os.path.basename(filename))[0]
1706 1706 else:
1707 1707 name = '__main__'
1708 1708
1709 1709 main_mod = self.shell.new_main_mod()
1710 1710 prog_ns = main_mod.__dict__
1711 1711 prog_ns['__name__'] = name
1712 1712
1713 1713 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
1714 1714 # set the __file__ global in the script's namespace
1715 1715 prog_ns['__file__'] = filename
1716 1716
1717 1717 # pickle fix. See interactiveshell for an explanation. But we need to make sure
1718 1718 # that, if we overwrite __main__, we replace it at the end
1719 1719 main_mod_name = prog_ns['__name__']
1720 1720
1721 1721 if main_mod_name == '__main__':
1722 1722 restore_main = sys.modules['__main__']
1723 1723 else:
1724 1724 restore_main = False
1725 1725
1726 1726 # This needs to be undone at the end to prevent holding references to
1727 1727 # every single object ever created.
1728 1728 sys.modules[main_mod_name] = main_mod
1729 1729
1730 1730 try:
1731 1731 stats = None
1732 1732 with self.shell.readline_no_record:
1733 1733 if 'p' in opts:
1734 1734 stats = self.magic_prun('', 0, opts, arg_lst, prog_ns)
1735 1735 else:
1736 1736 if 'd' in opts:
1737 1737 deb = debugger.Pdb(self.shell.colors)
1738 1738 # reset Breakpoint state, which is moronically kept
1739 1739 # in a class
1740 1740 bdb.Breakpoint.next = 1
1741 1741 bdb.Breakpoint.bplist = {}
1742 1742 bdb.Breakpoint.bpbynumber = [None]
1743 1743 # Set an initial breakpoint to stop execution
1744 1744 maxtries = 10
1745 1745 bp = int(opts.get('b', [1])[0])
1746 1746 checkline = deb.checkline(filename, bp)
1747 1747 if not checkline:
1748 1748 for bp in range(bp + 1, bp + maxtries + 1):
1749 1749 if deb.checkline(filename, bp):
1750 1750 break
1751 1751 else:
1752 1752 msg = ("\nI failed to find a valid line to set "
1753 1753 "a breakpoint\n"
1754 1754 "after trying up to line: %s.\n"
1755 1755 "Please set a valid breakpoint manually "
1756 1756 "with the -b option." % bp)
1757 1757 error(msg)
1758 1758 return
1759 1759 # if we find a good linenumber, set the breakpoint
1760 1760 deb.do_break('%s:%s' % (filename, bp))
1761 1761 # Start file run
1762 1762 print "NOTE: Enter 'c' at the",
1763 1763 print "%s prompt to start your script." % deb.prompt
1764 1764 ns = {'execfile': py3compat.execfile, 'prog_ns': prog_ns}
1765 1765 try:
1766 1766 deb.run('execfile("%s", prog_ns)' % filename, ns)
1767 1767
1768 1768 except:
1769 1769 etype, value, tb = sys.exc_info()
1770 1770 # Skip three frames in the traceback: the %run one,
1771 1771 # one inside bdb.py, and the command-line typed by the
1772 1772 # user (run by exec in pdb itself).
1773 1773 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
1774 1774 else:
1775 1775 if runner is None:
1776 1776 runner = self.default_runner
1777 1777 if runner is None:
1778 1778 runner = self.shell.safe_execfile
1779 1779 if 't' in opts:
1780 1780 # timed execution
1781 1781 try:
1782 1782 nruns = int(opts['N'][0])
1783 1783 if nruns < 1:
1784 1784 error('Number of runs must be >=1')
1785 1785 return
1786 1786 except (KeyError):
1787 1787 nruns = 1
1788 1788 twall0 = time.time()
1789 1789 if nruns == 1:
1790 1790 t0 = clock2()
1791 1791 runner(filename, prog_ns, prog_ns,
1792 1792 exit_ignore=exit_ignore)
1793 1793 t1 = clock2()
1794 1794 t_usr = t1[0] - t0[0]
1795 1795 t_sys = t1[1] - t0[1]
1796 1796 print "\nIPython CPU timings (estimated):"
1797 1797 print " User : %10.2f s." % t_usr
1798 1798 print " System : %10.2f s." % t_sys
1799 1799 else:
1800 1800 runs = range(nruns)
1801 1801 t0 = clock2()
1802 1802 for nr in runs:
1803 1803 runner(filename, prog_ns, prog_ns,
1804 1804 exit_ignore=exit_ignore)
1805 1805 t1 = clock2()
1806 1806 t_usr = t1[0] - t0[0]
1807 1807 t_sys = t1[1] - t0[1]
1808 1808 print "\nIPython CPU timings (estimated):"
1809 1809 print "Total runs performed:", nruns
1810 1810 print " Times : %10.2f %10.2f" % ('Total', 'Per run')
1811 1811 print " User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
1812 1812 print " System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
1813 1813 twall1 = time.time()
1814 1814 print "Wall time: %10.2f s." % (twall1 - twall0)
1815 1815
1816 1816 else:
1817 1817 # regular execution
1818 1818 runner(filename, prog_ns, prog_ns, exit_ignore=exit_ignore)
1819 1819
1820 1820 if 'i' in opts:
1821 1821 self.shell.user_ns['__name__'] = __name__save
1822 1822 else:
1823 1823 # The shell MUST hold a reference to prog_ns so after %run
1824 1824 # exits, the python deletion mechanism doesn't zero it out
1825 1825 # (leaving dangling references).
1826 1826 self.shell.cache_main_mod(prog_ns, filename)
1827 1827 # update IPython interactive namespace
1828 1828
1829 1829 # Some forms of read errors on the file may mean the
1830 1830 # __name__ key was never set; using pop we don't have to
1831 1831 # worry about a possible KeyError.
1832 1832 prog_ns.pop('__name__', None)
1833 1833
1834 1834 self.shell.user_ns.update(prog_ns)
1835 1835 finally:
1836 1836 # It's a bit of a mystery why, but __builtins__ can change from
1837 1837 # being a module to becoming a dict missing some key data after
1838 1838 # %run. As best I can see, this is NOT something IPython is doing
1839 1839 # at all, and similar problems have been reported before:
1840 1840 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
1841 1841 # Since this seems to be done by the interpreter itself, the best
1842 1842 # we can do is to at least restore __builtins__ for the user on
1843 1843 # exit.
1844 1844 self.shell.user_ns['__builtins__'] = builtin_mod
1845 1845
1846 1846 # Ensure key global structures are restored
1847 1847 sys.argv = save_argv
1848 1848 if restore_main:
1849 1849 sys.modules['__main__'] = restore_main
1850 1850 else:
1851 1851 # Remove from sys.modules the reference to main_mod we'd
1852 1852 # added. Otherwise it will trap references to objects
1853 1853 # contained therein.
1854 1854 del sys.modules[main_mod_name]
1855 1855
1856 1856 return stats
1857 1857
1858 1858 @skip_doctest
1859 1859 def magic_timeit(self, parameter_s =''):
1860 1860 """Time execution of a Python statement or expression
1861 1861
1862 1862 Usage:\\
1863 1863 %timeit [-n<N> -r<R> [-t|-c]] statement
1864 1864
1865 1865 Time execution of a Python statement or expression using the timeit
1866 1866 module.
1867 1867
1868 1868 Options:
1869 1869 -n<N>: execute the given statement <N> times in a loop. If this value
1870 1870 is not given, a fitting value is chosen.
1871 1871
1872 1872 -r<R>: repeat the loop iteration <R> times and take the best result.
1873 1873 Default: 3
1874 1874
1875 1875 -t: use time.time to measure the time, which is the default on Unix.
1876 1876 This function measures wall time.
1877 1877
1878 1878 -c: use time.clock to measure the time, which is the default on
1879 1879 Windows and measures wall time. On Unix, resource.getrusage is used
1880 1880 instead and returns the CPU user time.
1881 1881
1882 1882 -p<P>: use a precision of <P> digits to display the timing result.
1883 1883 Default: 3
1884 1884
1885 1885
1886 1886 Examples
1887 1887 --------
1888 1888 ::
1889 1889
1890 1890 In [1]: %timeit pass
1891 1891 10000000 loops, best of 3: 53.3 ns per loop
1892 1892
1893 1893 In [2]: u = None
1894 1894
1895 1895 In [3]: %timeit u is None
1896 1896 10000000 loops, best of 3: 184 ns per loop
1897 1897
1898 1898 In [4]: %timeit -r 4 u == None
1899 1899 1000000 loops, best of 4: 242 ns per loop
1900 1900
1901 1901 In [5]: import time
1902 1902
1903 1903 In [6]: %timeit -n1 time.sleep(2)
1904 1904 1 loops, best of 3: 2 s per loop
1905 1905
1906 1906
1907 1907 The times reported by %timeit will be slightly higher than those
1908 1908 reported by the timeit.py script when variables are accessed. This is
1909 1909 due to the fact that %timeit executes the statement in the namespace
1910 1910 of the shell, compared with timeit.py, which uses a single setup
1911 1911 statement to import function or create variables. Generally, the bias
1912 1912 does not matter as long as results from timeit.py are not mixed with
1913 1913 those from %timeit."""
1914 1914
1915 1915 import timeit
1916 1916 import math
1917 1917
1918 1918 # XXX: Unfortunately the unicode 'micro' symbol can cause problems in
1919 1919 # certain terminals. Until we figure out a robust way of
1920 1920 # auto-detecting if the terminal can deal with it, use plain 'us' for
1921 1921 # microseconds. I am really NOT happy about disabling the proper
1922 1922 # 'micro' prefix, but crashing is worse... If anyone knows what the
1923 1923 # right solution for this is, I'm all ears...
1924 1924 #
1925 1925 # Note: using
1926 1926 #
1927 1927 # s = u'\xb5'
1928 1928 # s.encode(sys.getdefaultencoding())
1929 1929 #
1930 1930 # is not sufficient, as I've seen terminals where that fails but
1931 1931 # print s
1932 1932 #
1933 1933 # succeeds
1934 1934 #
1935 1935 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1936 1936
1937 1937 #units = [u"s", u"ms",u'\xb5',"ns"]
1938 1938 units = [u"s", u"ms",u'us',"ns"]
1939 1939
1940 1940 scaling = [1, 1e3, 1e6, 1e9]
1941 1941
1942 1942 opts, stmt = self.parse_options(parameter_s,'n:r:tcp:',
1943 1943 posix=False, strict=False)
1944 1944 if stmt == "":
1945 1945 return
1946 1946 timefunc = timeit.default_timer
1947 1947 number = int(getattr(opts, "n", 0))
1948 1948 repeat = int(getattr(opts, "r", timeit.default_repeat))
1949 1949 precision = int(getattr(opts, "p", 3))
1950 1950 if hasattr(opts, "t"):
1951 1951 timefunc = time.time
1952 1952 if hasattr(opts, "c"):
1953 1953 timefunc = clock
1954 1954
1955 1955 timer = timeit.Timer(timer=timefunc)
1956 1956 # this code has tight coupling to the inner workings of timeit.Timer,
1957 1957 # but is there a better way to achieve that the code stmt has access
1958 1958 # to the shell namespace?
1959 1959
1960 1960 src = timeit.template % {'stmt': timeit.reindent(stmt, 8),
1961 1961 'setup': "pass"}
1962 1962 # Track compilation time so it can be reported if too long
1963 1963 # Minimum time above which compilation time will be reported
1964 1964 tc_min = 0.1
1965 1965
1966 1966 t0 = clock()
1967 1967 code = compile(src, "<magic-timeit>", "exec")
1968 1968 tc = clock()-t0
1969 1969
1970 1970 ns = {}
1971 1971 exec code in self.shell.user_ns, ns
1972 1972 timer.inner = ns["inner"]
1973 1973
1974 1974 if number == 0:
1975 1975 # determine number so that 0.2 <= total time < 2.0
1976 1976 number = 1
1977 1977 for i in range(1, 10):
1978 1978 if timer.timeit(number) >= 0.2:
1979 1979 break
1980 1980 number *= 10
1981 1981
1982 1982 best = min(timer.repeat(repeat, number)) / number
1983 1983
1984 1984 if best > 0.0 and best < 1000.0:
1985 1985 order = min(-int(math.floor(math.log10(best)) // 3), 3)
1986 1986 elif best >= 1000.0:
1987 1987 order = 0
1988 1988 else:
1989 1989 order = 3
1990 1990 print u"%d loops, best of %d: %.*g %s per loop" % (number, repeat,
1991 1991 precision,
1992 1992 best * scaling[order],
1993 1993 units[order])
1994 1994 if tc > tc_min:
1995 1995 print "Compiler time: %.2f s" % tc
1996 1996
1997 1997 @skip_doctest
1998 1998 @needs_local_scope
1999 1999 def magic_time(self,parameter_s, user_locals):
2000 2000 """Time execution of a Python statement or expression.
2001 2001
2002 2002 The CPU and wall clock times are printed, and the value of the
2003 2003 expression (if any) is returned. Note that under Win32, system time
2004 2004 is always reported as 0, since it can not be measured.
2005 2005
2006 2006 This function provides very basic timing functionality. In Python
2007 2007 2.3, the timeit module offers more control and sophistication, so this
2008 2008 could be rewritten to use it (patches welcome).
2009 2009
2010 2010 Examples
2011 2011 --------
2012 2012 ::
2013 2013
2014 2014 In [1]: time 2**128
2015 2015 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
2016 2016 Wall time: 0.00
2017 2017 Out[1]: 340282366920938463463374607431768211456L
2018 2018
2019 2019 In [2]: n = 1000000
2020 2020
2021 2021 In [3]: time sum(range(n))
2022 2022 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
2023 2023 Wall time: 1.37
2024 2024 Out[3]: 499999500000L
2025 2025
2026 2026 In [4]: time print 'hello world'
2027 2027 hello world
2028 2028 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
2029 2029 Wall time: 0.00
2030 2030
2031 2031 Note that the time needed by Python to compile the given expression
2032 2032 will be reported if it is more than 0.1s. In this example, the
2033 2033 actual exponentiation is done by Python at compilation time, so while
2034 2034 the expression can take a noticeable amount of time to compute, that
2035 2035 time is purely due to the compilation:
2036 2036
2037 2037 In [5]: time 3**9999;
2038 2038 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
2039 2039 Wall time: 0.00 s
2040 2040
2041 2041 In [6]: time 3**999999;
2042 2042 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
2043 2043 Wall time: 0.00 s
2044 2044 Compiler : 0.78 s
2045 2045 """
2046 2046
2047 2047 # fail immediately if the given expression can't be compiled
2048 2048
2049 2049 expr = self.shell.prefilter(parameter_s,False)
2050 2050
2051 2051 # Minimum time above which compilation time will be reported
2052 2052 tc_min = 0.1
2053 2053
2054 2054 try:
2055 2055 mode = 'eval'
2056 2056 t0 = clock()
2057 2057 code = compile(expr,'<timed eval>',mode)
2058 2058 tc = clock()-t0
2059 2059 except SyntaxError:
2060 2060 mode = 'exec'
2061 2061 t0 = clock()
2062 2062 code = compile(expr,'<timed exec>',mode)
2063 2063 tc = clock()-t0
2064 2064 # skew measurement as little as possible
2065 2065 glob = self.shell.user_ns
2066 2066 wtime = time.time
2067 2067 # time execution
2068 2068 wall_st = wtime()
2069 2069 if mode=='eval':
2070 2070 st = clock2()
2071 2071 out = eval(code, glob, user_locals)
2072 2072 end = clock2()
2073 2073 else:
2074 2074 st = clock2()
2075 2075 exec code in glob, user_locals
2076 2076 end = clock2()
2077 2077 out = None
2078 2078 wall_end = wtime()
2079 2079 # Compute actual times and report
2080 2080 wall_time = wall_end-wall_st
2081 2081 cpu_user = end[0]-st[0]
2082 2082 cpu_sys = end[1]-st[1]
2083 2083 cpu_tot = cpu_user+cpu_sys
2084 2084 print "CPU times: user %.2f s, sys: %.2f s, total: %.2f s" % \
2085 2085 (cpu_user,cpu_sys,cpu_tot)
2086 2086 print "Wall time: %.2f s" % wall_time
2087 2087 if tc > tc_min:
2088 2088 print "Compiler : %.2f s" % tc
2089 2089 return out
2090 2090
2091 2091 @skip_doctest
2092 2092 def magic_macro(self,parameter_s = ''):
2093 2093 """Define a macro for future re-execution. It accepts ranges of history,
2094 2094 filenames or string objects.
2095 2095
2096 2096 Usage:\\
2097 2097 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
2098 2098
2099 2099 Options:
2100 2100
2101 2101 -r: use 'raw' input. By default, the 'processed' history is used,
2102 2102 so that magics are loaded in their transformed version to valid
2103 2103 Python. If this option is given, the raw input as typed as the
2104 2104 command line is used instead.
2105 2105
2106 2106 This will define a global variable called `name` which is a string
2107 2107 made of joining the slices and lines you specify (n1,n2,... numbers
2108 2108 above) from your input history into a single string. This variable
2109 2109 acts like an automatic function which re-executes those lines as if
2110 2110 you had typed them. You just type 'name' at the prompt and the code
2111 2111 executes.
2112 2112
2113 2113 The syntax for indicating input ranges is described in %history.
2114 2114
2115 2115 Note: as a 'hidden' feature, you can also use traditional python slice
2116 2116 notation, where N:M means numbers N through M-1.
2117 2117
2118 2118 For example, if your history contains (%hist prints it)::
2119 2119
2120 2120 44: x=1
2121 2121 45: y=3
2122 2122 46: z=x+y
2123 2123 47: print x
2124 2124 48: a=5
2125 2125 49: print 'x',x,'y',y
2126 2126
2127 2127 you can create a macro with lines 44 through 47 (included) and line 49
2128 2128 called my_macro with::
2129 2129
2130 2130 In [55]: %macro my_macro 44-47 49
2131 2131
2132 2132 Now, typing `my_macro` (without quotes) will re-execute all this code
2133 2133 in one pass.
2134 2134
2135 2135 You don't need to give the line-numbers in order, and any given line
2136 2136 number can appear multiple times. You can assemble macros with any
2137 2137 lines from your input history in any order.
2138 2138
2139 2139 The macro is a simple object which holds its value in an attribute,
2140 2140 but IPython's display system checks for macros and executes them as
2141 2141 code instead of printing them when you type their name.
2142 2142
2143 2143 You can view a macro's contents by explicitly printing it with::
2144 2144
2145 2145 print macro_name
2146 2146
2147 2147 """
2148 2148 opts,args = self.parse_options(parameter_s,'r',mode='list')
2149 2149 if not args: # List existing macros
2150 2150 return sorted(k for k,v in self.shell.user_ns.iteritems() if\
2151 2151 isinstance(v, Macro))
2152 2152 if len(args) == 1:
2153 2153 raise UsageError(
2154 2154 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
2155 2155 name, codefrom = args[0], " ".join(args[1:])
2156 2156
2157 2157 #print 'rng',ranges # dbg
2158 2158 try:
2159 2159 lines = self.shell.find_user_code(codefrom, 'r' in opts)
2160 2160 except (ValueError, TypeError) as e:
2161 2161 print e.args[0]
2162 2162 return
2163 2163 macro = Macro(lines)
2164 2164 self.shell.define_macro(name, macro)
2165 2165 print 'Macro `%s` created. To execute, type its name (without quotes).' % name
2166 2166 print '=== Macro contents: ==='
2167 2167 print macro,
2168 2168
2169 2169 def magic_save(self,parameter_s = ''):
2170 2170 """Save a set of lines or a macro to a given filename.
2171 2171
2172 2172 Usage:\\
2173 2173 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
2174 2174
2175 2175 Options:
2176 2176
2177 2177 -r: use 'raw' input. By default, the 'processed' history is used,
2178 2178 so that magics are loaded in their transformed version to valid
2179 2179 Python. If this option is given, the raw input as typed as the
2180 2180 command line is used instead.
2181 2181
2182 2182 This function uses the same syntax as %history for input ranges,
2183 2183 then saves the lines to the filename you specify.
2184 2184
2185 2185 It adds a '.py' extension to the file if you don't do so yourself, and
2186 2186 it asks for confirmation before overwriting existing files."""
2187 2187
2188 2188 opts,args = self.parse_options(parameter_s,'r',mode='list')
2189 2189 fname, codefrom = unquote_filename(args[0]), " ".join(args[1:])
2190 2190 if not fname.endswith('.py'):
2191 2191 fname += '.py'
2192 2192 if os.path.isfile(fname):
2193 2193 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
2194 2194 if not overwrite :
2195 2195 print 'Operation cancelled.'
2196 2196 return
2197 2197 try:
2198 2198 cmds = self.shell.find_user_code(codefrom, 'r' in opts)
2199 2199 except (TypeError, ValueError) as e:
2200 2200 print e.args[0]
2201 2201 return
2202 2202 with io.open(fname,'w', encoding="utf-8") as f:
2203 2203 f.write(u"# coding: utf-8\n")
2204 2204 f.write(py3compat.cast_unicode(cmds))
2205 2205 print 'The following commands were written to file `%s`:' % fname
2206 2206 print cmds
2207 2207
2208 2208 def magic_pastebin(self, parameter_s = ''):
2209 2209 """Upload code to Github's Gist paste bin, returning the URL.
2210 2210
2211 2211 Usage:\\
2212 2212 %pastebin [-d "Custom description"] 1-7
2213 2213
2214 2214 The argument can be an input history range, a filename, or the name of a
2215 2215 string or macro.
2216 2216
2217 2217 Options:
2218 2218
2219 2219 -d: Pass a custom description for the gist. The default will say
2220 2220 "Pasted from IPython".
2221 2221 """
2222 2222 opts, args = self.parse_options(parameter_s, 'd:')
2223 2223
2224 2224 try:
2225 2225 code = self.shell.find_user_code(args)
2226 2226 except (ValueError, TypeError) as e:
2227 2227 print e.args[0]
2228 2228 return
2229 2229
2230 2230 post_data = json.dumps({
2231 2231 "description": opts.get('d', "Pasted from IPython"),
2232 2232 "public": True,
2233 2233 "files": {
2234 2234 "file1.py": {
2235 2235 "content": code
2236 2236 }
2237 2237 }
2238 2238 }).encode('utf-8')
2239 2239
2240 2240 response = urlopen("https://api.github.com/gists", post_data)
2241 2241 response_data = json.loads(response.read().decode('utf-8'))
2242 2242 return response_data['html_url']
2243 2243
2244 2244 def magic_loadpy(self, arg_s):
2245 2245 """Alias of `%load`
2246 2246
2247 2247 `%loadpy` has gained some flexibility and droped the requirement of a `.py`
2248 2248 extension. So it has been renamed simply into %load. You can look at
2249 2249 `%load`'s docstring for more info.
2250 2250 """
2251 2251 self.magic_load(arg_s)
2252 2252
2253 2253 def magic_load(self, arg_s):
2254 2254 """Load code into the current frontend.
2255 2255
2256 2256 Usage:\\
2257 2257 %load [options] source
2258 2258
2259 2259 where source can be a filename, URL, input history range or macro
2260 2260
2261 2261 Options:
2262 2262 --------
2263 2263 -y : Don't ask confirmation for loading source above 200 000 characters.
2264 2264
2265 2265 This magic command can either take a local filename, a URL, an history
2266 2266 range (see %history) or a macro as argument, it will prompt for
2267 2267 confirmation before loading source with more than 200 000 characters, unless
2268 2268 -y flag is passed or if the frontend does not support raw_input::
2269 2269
2270 2270 %load myscript.py
2271 2271 %load 7-27
2272 2272 %load myMacro
2273 2273 %load http://www.example.com/myscript.py
2274 2274 """
2275 2275 opts,args = self.parse_options(arg_s,'y')
2276 2276
2277 2277 contents = self.shell.find_user_code(args)
2278 2278 l = len(contents)
2279 2279
2280 2280 # 200 000 is ~ 2500 full 80 caracter lines
2281 2281 # so in average, more than 5000 lines
2282 2282 if l > 200000 and 'y' not in opts:
2283 2283 try:
2284 2284 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
2285 2285 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
2286 2286 except StdinNotImplementedError:
2287 2287 #asume yes if raw input not implemented
2288 2288 ans = True
2289 2289
2290 2290 if ans is False :
2291 2291 print 'Operation cancelled.'
2292 2292 return
2293 2293
2294 2294 self.set_next_input(contents)
2295 2295
2296 2296 def _find_edit_target(self, args, opts, last_call):
2297 2297 """Utility method used by magic_edit to find what to edit."""
2298 2298
2299 2299 def make_filename(arg):
2300 2300 "Make a filename from the given args"
2301 2301 arg = unquote_filename(arg)
2302 2302 try:
2303 2303 filename = get_py_filename(arg)
2304 2304 except IOError:
2305 2305 # If it ends with .py but doesn't already exist, assume we want
2306 2306 # a new file.
2307 2307 if arg.endswith('.py'):
2308 2308 filename = arg
2309 2309 else:
2310 2310 filename = None
2311 2311 return filename
2312 2312
2313 2313 # Set a few locals from the options for convenience:
2314 2314 opts_prev = 'p' in opts
2315 2315 opts_raw = 'r' in opts
2316 2316
2317 2317 # custom exceptions
2318 2318 class DataIsObject(Exception): pass
2319 2319
2320 2320 # Default line number value
2321 2321 lineno = opts.get('n',None)
2322 2322
2323 2323 if opts_prev:
2324 2324 args = '_%s' % last_call[0]
2325 2325 if not self.shell.user_ns.has_key(args):
2326 2326 args = last_call[1]
2327 2327
2328 2328 # use last_call to remember the state of the previous call, but don't
2329 2329 # let it be clobbered by successive '-p' calls.
2330 2330 try:
2331 2331 last_call[0] = self.shell.displayhook.prompt_count
2332 2332 if not opts_prev:
2333 2333 last_call[1] = args
2334 2334 except:
2335 2335 pass
2336 2336
2337 2337 # by default this is done with temp files, except when the given
2338 2338 # arg is a filename
2339 2339 use_temp = True
2340 2340
2341 2341 data = ''
2342 2342
2343 2343 # First, see if the arguments should be a filename.
2344 2344 filename = make_filename(args)
2345 2345 if filename:
2346 2346 use_temp = False
2347 2347 elif args:
2348 2348 # Mode where user specifies ranges of lines, like in %macro.
2349 2349 data = self.shell.extract_input_lines(args, opts_raw)
2350 2350 if not data:
2351 2351 try:
2352 2352 # Load the parameter given as a variable. If not a string,
2353 2353 # process it as an object instead (below)
2354 2354
2355 2355 #print '*** args',args,'type',type(args) # dbg
2356 2356 data = eval(args, self.shell.user_ns)
2357 2357 if not isinstance(data, basestring):
2358 2358 raise DataIsObject
2359 2359
2360 2360 except (NameError,SyntaxError):
2361 2361 # given argument is not a variable, try as a filename
2362 2362 filename = make_filename(args)
2363 2363 if filename is None:
2364 2364 warn("Argument given (%s) can't be found as a variable "
2365 2365 "or as a filename." % args)
2366 2366 return
2367 2367 use_temp = False
2368 2368
2369 2369 except DataIsObject:
2370 2370 # macros have a special edit function
2371 2371 if isinstance(data, Macro):
2372 2372 raise MacroToEdit(data)
2373 2373
2374 2374 # For objects, try to edit the file where they are defined
2375 2375 try:
2376 2376 filename = inspect.getabsfile(data)
2377 2377 if 'fakemodule' in filename.lower() and inspect.isclass(data):
2378 2378 # class created by %edit? Try to find source
2379 2379 # by looking for method definitions instead, the
2380 2380 # __module__ in those classes is FakeModule.
2381 2381 attrs = [getattr(data, aname) for aname in dir(data)]
2382 2382 for attr in attrs:
2383 2383 if not inspect.ismethod(attr):
2384 2384 continue
2385 2385 filename = inspect.getabsfile(attr)
2386 2386 if filename and 'fakemodule' not in filename.lower():
2387 2387 # change the attribute to be the edit target instead
2388 2388 data = attr
2389 2389 break
2390 2390
2391 2391 datafile = 1
2392 2392 except TypeError:
2393 2393 filename = make_filename(args)
2394 2394 datafile = 1
2395 2395 warn('Could not find file where `%s` is defined.\n'
2396 2396 'Opening a file named `%s`' % (args,filename))
2397 2397 # Now, make sure we can actually read the source (if it was in
2398 2398 # a temp file it's gone by now).
2399 2399 if datafile:
2400 2400 try:
2401 2401 if lineno is None:
2402 2402 lineno = inspect.getsourcelines(data)[1]
2403 2403 except IOError:
2404 2404 filename = make_filename(args)
2405 2405 if filename is None:
2406 2406 warn('The file `%s` where `%s` was defined cannot '
2407 2407 'be read.' % (filename,data))
2408 2408 return
2409 2409 use_temp = False
2410 2410
2411 2411 if use_temp:
2412 2412 filename = self.shell.mktempfile(data)
2413 2413 print 'IPython will make a temporary file named:',filename
2414 2414
2415 2415 return filename, lineno, use_temp
2416 2416
2417 2417 def _edit_macro(self,mname,macro):
2418 2418 """open an editor with the macro data in a file"""
2419 2419 filename = self.shell.mktempfile(macro.value)
2420 2420 self.shell.hooks.editor(filename)
2421 2421
2422 2422 # and make a new macro object, to replace the old one
2423 2423 mfile = open(filename)
2424 2424 mvalue = mfile.read()
2425 2425 mfile.close()
2426 2426 self.shell.user_ns[mname] = Macro(mvalue)
2427 2427
2428 2428 def magic_ed(self,parameter_s=''):
2429 2429 """Alias to %edit."""
2430 2430 return self.magic_edit(parameter_s)
2431 2431
2432 2432 @skip_doctest
2433 2433 def magic_edit(self,parameter_s='',last_call=['','']):
2434 2434 """Bring up an editor and execute the resulting code.
2435 2435
2436 2436 Usage:
2437 2437 %edit [options] [args]
2438 2438
2439 2439 %edit runs IPython's editor hook. The default version of this hook is
2440 2440 set to call the editor specified by your $EDITOR environment variable.
2441 2441 If this isn't found, it will default to vi under Linux/Unix and to
2442 2442 notepad under Windows. See the end of this docstring for how to change
2443 2443 the editor hook.
2444 2444
2445 2445 You can also set the value of this editor via the
2446 2446 ``TerminalInteractiveShell.editor`` option in your configuration file.
2447 2447 This is useful if you wish to use a different editor from your typical
2448 2448 default with IPython (and for Windows users who typically don't set
2449 2449 environment variables).
2450 2450
2451 2451 This command allows you to conveniently edit multi-line code right in
2452 2452 your IPython session.
2453 2453
2454 2454 If called without arguments, %edit opens up an empty editor with a
2455 2455 temporary file and will execute the contents of this file when you
2456 2456 close it (don't forget to save it!).
2457 2457
2458 2458
2459 2459 Options:
2460 2460
2461 2461 -n <number>: open the editor at a specified line number. By default,
2462 2462 the IPython editor hook uses the unix syntax 'editor +N filename', but
2463 2463 you can configure this by providing your own modified hook if your
2464 2464 favorite editor supports line-number specifications with a different
2465 2465 syntax.
2466 2466
2467 2467 -p: this will call the editor with the same data as the previous time
2468 2468 it was used, regardless of how long ago (in your current session) it
2469 2469 was.
2470 2470
2471 2471 -r: use 'raw' input. This option only applies to input taken from the
2472 2472 user's history. By default, the 'processed' history is used, so that
2473 2473 magics are loaded in their transformed version to valid Python. If
2474 2474 this option is given, the raw input as typed as the command line is
2475 2475 used instead. When you exit the editor, it will be executed by
2476 2476 IPython's own processor.
2477 2477
2478 2478 -x: do not execute the edited code immediately upon exit. This is
2479 2479 mainly useful if you are editing programs which need to be called with
2480 2480 command line arguments, which you can then do using %run.
2481 2481
2482 2482
2483 2483 Arguments:
2484 2484
2485 2485 If arguments are given, the following possibilities exist:
2486 2486
2487 2487 - If the argument is a filename, IPython will load that into the
2488 2488 editor. It will execute its contents with execfile() when you exit,
2489 2489 loading any code in the file into your interactive namespace.
2490 2490
2491 2491 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
2492 2492 The syntax is the same as in the %history magic.
2493 2493
2494 2494 - If the argument is a string variable, its contents are loaded
2495 2495 into the editor. You can thus edit any string which contains
2496 2496 python code (including the result of previous edits).
2497 2497
2498 2498 - If the argument is the name of an object (other than a string),
2499 2499 IPython will try to locate the file where it was defined and open the
2500 2500 editor at the point where it is defined. You can use `%edit function`
2501 2501 to load an editor exactly at the point where 'function' is defined,
2502 2502 edit it and have the file be executed automatically.
2503 2503
2504 2504 - If the object is a macro (see %macro for details), this opens up your
2505 2505 specified editor with a temporary file containing the macro's data.
2506 2506 Upon exit, the macro is reloaded with the contents of the file.
2507 2507
2508 2508 Note: opening at an exact line is only supported under Unix, and some
2509 2509 editors (like kedit and gedit up to Gnome 2.8) do not understand the
2510 2510 '+NUMBER' parameter necessary for this feature. Good editors like
2511 2511 (X)Emacs, vi, jed, pico and joe all do.
2512 2512
2513 2513 After executing your code, %edit will return as output the code you
2514 2514 typed in the editor (except when it was an existing file). This way
2515 2515 you can reload the code in further invocations of %edit as a variable,
2516 2516 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
2517 2517 the output.
2518 2518
2519 2519 Note that %edit is also available through the alias %ed.
2520 2520
2521 2521 This is an example of creating a simple function inside the editor and
2522 2522 then modifying it. First, start up the editor::
2523 2523
2524 2524 In [1]: ed
2525 2525 Editing... done. Executing edited code...
2526 2526 Out[1]: 'def foo():\\n print "foo() was defined in an editing
2527 2527 session"\\n'
2528 2528
2529 2529 We can then call the function foo()::
2530 2530
2531 2531 In [2]: foo()
2532 2532 foo() was defined in an editing session
2533 2533
2534 2534 Now we edit foo. IPython automatically loads the editor with the
2535 2535 (temporary) file where foo() was previously defined::
2536 2536
2537 2537 In [3]: ed foo
2538 2538 Editing... done. Executing edited code...
2539 2539
2540 2540 And if we call foo() again we get the modified version::
2541 2541
2542 2542 In [4]: foo()
2543 2543 foo() has now been changed!
2544 2544
2545 2545 Here is an example of how to edit a code snippet successive
2546 2546 times. First we call the editor::
2547 2547
2548 2548 In [5]: ed
2549 2549 Editing... done. Executing edited code...
2550 2550 hello
2551 2551 Out[5]: "print 'hello'\\n"
2552 2552
2553 2553 Now we call it again with the previous output (stored in _)::
2554 2554
2555 2555 In [6]: ed _
2556 2556 Editing... done. Executing edited code...
2557 2557 hello world
2558 2558 Out[6]: "print 'hello world'\\n"
2559 2559
2560 2560 Now we call it with the output #8 (stored in _8, also as Out[8])::
2561 2561
2562 2562 In [7]: ed _8
2563 2563 Editing... done. Executing edited code...
2564 2564 hello again
2565 2565 Out[7]: "print 'hello again'\\n"
2566 2566
2567 2567
2568 2568 Changing the default editor hook:
2569 2569
2570 2570 If you wish to write your own editor hook, you can put it in a
2571 2571 configuration file which you load at startup time. The default hook
2572 2572 is defined in the IPython.core.hooks module, and you can use that as a
2573 2573 starting example for further modifications. That file also has
2574 2574 general instructions on how to set a new hook for use once you've
2575 2575 defined it."""
2576 2576 opts,args = self.parse_options(parameter_s,'prxn:')
2577 2577
2578 2578 try:
2579 2579 filename, lineno, is_temp = self._find_edit_target(args, opts, last_call)
2580 2580 except MacroToEdit as e:
2581 2581 self._edit_macro(args, e.args[0])
2582 2582 return
2583 2583
2584 2584 # do actual editing here
2585 2585 print 'Editing...',
2586 2586 sys.stdout.flush()
2587 2587 try:
2588 2588 # Quote filenames that may have spaces in them
2589 2589 if ' ' in filename:
2590 2590 filename = "'%s'" % filename
2591 2591 self.shell.hooks.editor(filename,lineno)
2592 2592 except TryNext:
2593 2593 warn('Could not open editor')
2594 2594 return
2595 2595
2596 2596 # XXX TODO: should this be generalized for all string vars?
2597 2597 # For now, this is special-cased to blocks created by cpaste
2598 2598 if args.strip() == 'pasted_block':
2599 2599 self.shell.user_ns['pasted_block'] = file_read(filename)
2600 2600
2601 2601 if 'x' in opts: # -x prevents actual execution
2602 2602 print
2603 2603 else:
2604 2604 print 'done. Executing edited code...'
2605 2605 if 'r' in opts: # Untranslated IPython code
2606 2606 self.shell.run_cell(file_read(filename),
2607 2607 store_history=False)
2608 2608 else:
2609 2609 self.shell.safe_execfile(filename, self.shell.user_ns,
2610 2610 self.shell.user_ns)
2611 2611
2612 2612 if is_temp:
2613 2613 try:
2614 2614 return open(filename).read()
2615 2615 except IOError,msg:
2616 2616 if msg.filename == filename:
2617 2617 warn('File not found. Did you forget to save?')
2618 2618 return
2619 2619 else:
2620 2620 self.shell.showtraceback()
2621 2621
2622 2622 def magic_xmode(self,parameter_s = ''):
2623 2623 """Switch modes for the exception handlers.
2624 2624
2625 2625 Valid modes: Plain, Context and Verbose.
2626 2626
2627 2627 If called without arguments, acts as a toggle."""
2628 2628
2629 2629 def xmode_switch_err(name):
2630 2630 warn('Error changing %s exception modes.\n%s' %
2631 2631 (name,sys.exc_info()[1]))
2632 2632
2633 2633 shell = self.shell
2634 2634 new_mode = parameter_s.strip().capitalize()
2635 2635 try:
2636 2636 shell.InteractiveTB.set_mode(mode=new_mode)
2637 2637 print 'Exception reporting mode:',shell.InteractiveTB.mode
2638 2638 except:
2639 2639 xmode_switch_err('user')
2640 2640
2641 2641 def magic_colors(self,parameter_s = ''):
2642 2642 """Switch color scheme for prompts, info system and exception handlers.
2643 2643
2644 2644 Currently implemented schemes: NoColor, Linux, LightBG.
2645 2645
2646 2646 Color scheme names are not case-sensitive.
2647 2647
2648 2648 Examples
2649 2649 --------
2650 2650 To get a plain black and white terminal::
2651 2651
2652 2652 %colors nocolor
2653 2653 """
2654 2654
2655 2655 def color_switch_err(name):
2656 2656 warn('Error changing %s color schemes.\n%s' %
2657 2657 (name,sys.exc_info()[1]))
2658 2658
2659 2659
2660 2660 new_scheme = parameter_s.strip()
2661 2661 if not new_scheme:
2662 2662 raise UsageError(
2663 2663 "%colors: you must specify a color scheme. See '%colors?'")
2664 2664 return
2665 2665 # local shortcut
2666 2666 shell = self.shell
2667 2667
2668 2668 import IPython.utils.rlineimpl as readline
2669 2669
2670 2670 if not shell.colors_force and \
2671 2671 not readline.have_readline and sys.platform == "win32":
2672 2672 msg = """\
2673 2673 Proper color support under MS Windows requires the pyreadline library.
2674 2674 You can find it at:
2675 2675 http://ipython.org/pyreadline.html
2676 2676 Gary's readline needs the ctypes module, from:
2677 2677 http://starship.python.net/crew/theller/ctypes
2678 2678 (Note that ctypes is already part of Python versions 2.5 and newer).
2679 2679
2680 2680 Defaulting color scheme to 'NoColor'"""
2681 2681 new_scheme = 'NoColor'
2682 2682 warn(msg)
2683 2683
2684 2684 # readline option is 0
2685 2685 if not shell.colors_force and not shell.has_readline:
2686 2686 new_scheme = 'NoColor'
2687 2687
2688 2688 # Set prompt colors
2689 2689 try:
2690 2690 shell.prompt_manager.color_scheme = new_scheme
2691 2691 except:
2692 2692 color_switch_err('prompt')
2693 2693 else:
2694 2694 shell.colors = \
2695 2695 shell.prompt_manager.color_scheme_table.active_scheme_name
2696 2696 # Set exception colors
2697 2697 try:
2698 2698 shell.InteractiveTB.set_colors(scheme = new_scheme)
2699 2699 shell.SyntaxTB.set_colors(scheme = new_scheme)
2700 2700 except:
2701 2701 color_switch_err('exception')
2702 2702
2703 2703 # Set info (for 'object?') colors
2704 2704 if shell.color_info:
2705 2705 try:
2706 2706 shell.inspector.set_active_scheme(new_scheme)
2707 2707 except:
2708 2708 color_switch_err('object inspector')
2709 2709 else:
2710 2710 shell.inspector.set_active_scheme('NoColor')
2711 2711
2712 2712 def magic_pprint(self, parameter_s=''):
2713 2713 """Toggle pretty printing on/off."""
2714 2714 ptformatter = self.shell.display_formatter.formatters['text/plain']
2715 2715 ptformatter.pprint = bool(1 - ptformatter.pprint)
2716 2716 print 'Pretty printing has been turned', \
2717 2717 ['OFF','ON'][ptformatter.pprint]
2718 2718
2719 2719 #......................................................................
2720 2720 # Functions to implement unix shell-type things
2721 2721
2722 2722 @skip_doctest
2723 2723 def magic_alias(self, parameter_s = ''):
2724 2724 """Define an alias for a system command.
2725 2725
2726 2726 '%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
2727 2727
2728 2728 Then, typing 'alias_name params' will execute the system command 'cmd
2729 2729 params' (from your underlying operating system).
2730 2730
2731 2731 Aliases have lower precedence than magic functions and Python normal
2732 2732 variables, so if 'foo' is both a Python variable and an alias, the
2733 2733 alias can not be executed until 'del foo' removes the Python variable.
2734 2734
2735 2735 You can use the %l specifier in an alias definition to represent the
2736 2736 whole line when the alias is called. For example::
2737 2737
2738 2738 In [2]: alias bracket echo "Input in brackets: <%l>"
2739 2739 In [3]: bracket hello world
2740 2740 Input in brackets: <hello world>
2741 2741
2742 2742 You can also define aliases with parameters using %s specifiers (one
2743 2743 per parameter)::
2744 2744
2745 2745 In [1]: alias parts echo first %s second %s
2746 2746 In [2]: %parts A B
2747 2747 first A second B
2748 2748 In [3]: %parts A
2749 2749 Incorrect number of arguments: 2 expected.
2750 2750 parts is an alias to: 'echo first %s second %s'
2751 2751
2752 2752 Note that %l and %s are mutually exclusive. You can only use one or
2753 2753 the other in your aliases.
2754 2754
2755 2755 Aliases expand Python variables just like system calls using ! or !!
2756 2756 do: all expressions prefixed with '$' get expanded. For details of
2757 2757 the semantic rules, see PEP-215:
2758 2758 http://www.python.org/peps/pep-0215.html. This is the library used by
2759 2759 IPython for variable expansion. If you want to access a true shell
2760 2760 variable, an extra $ is necessary to prevent its expansion by
2761 2761 IPython::
2762 2762
2763 2763 In [6]: alias show echo
2764 2764 In [7]: PATH='A Python string'
2765 2765 In [8]: show $PATH
2766 2766 A Python string
2767 2767 In [9]: show $$PATH
2768 2768 /usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
2769 2769
2770 2770 You can use the alias facility to acess all of $PATH. See the %rehash
2771 2771 and %rehashx functions, which automatically create aliases for the
2772 2772 contents of your $PATH.
2773 2773
2774 2774 If called with no parameters, %alias prints the current alias table."""
2775 2775
2776 2776 par = parameter_s.strip()
2777 2777 if not par:
2778 2778 stored = self.shell.db.get('stored_aliases', {} )
2779 2779 aliases = sorted(self.shell.alias_manager.aliases)
2780 2780 # for k, v in stored:
2781 2781 # atab.append(k, v[0])
2782 2782
2783 2783 print "Total number of aliases:", len(aliases)
2784 2784 sys.stdout.flush()
2785 2785 return aliases
2786 2786
2787 2787 # Now try to define a new one
2788 2788 try:
2789 2789 alias,cmd = par.split(None, 1)
2790 2790 except:
2791 2791 print oinspect.getdoc(self.magic_alias)
2792 2792 else:
2793 2793 self.shell.alias_manager.soft_define_alias(alias, cmd)
2794 2794 # end magic_alias
2795 2795
2796 2796 def magic_unalias(self, parameter_s = ''):
2797 2797 """Remove an alias"""
2798 2798
2799 2799 aname = parameter_s.strip()
2800 2800 self.shell.alias_manager.undefine_alias(aname)
2801 2801 stored = self.shell.db.get('stored_aliases', {} )
2802 2802 if aname in stored:
2803 2803 print "Removing %stored alias",aname
2804 2804 del stored[aname]
2805 2805 self.shell.db['stored_aliases'] = stored
2806 2806
2807 2807 def magic_rehashx(self, parameter_s = ''):
2808 2808 """Update the alias table with all executable files in $PATH.
2809 2809
2810 2810 This version explicitly checks that every entry in $PATH is a file
2811 2811 with execute access (os.X_OK), so it is much slower than %rehash.
2812 2812
2813 2813 Under Windows, it checks executability as a match against a
2814 2814 '|'-separated string of extensions, stored in the IPython config
2815 2815 variable win_exec_ext. This defaults to 'exe|com|bat'.
2816 2816
2817 2817 This function also resets the root module cache of module completer,
2818 2818 used on slow filesystems.
2819 2819 """
2820 2820 from IPython.core.alias import InvalidAliasError
2821 2821
2822 2822 # for the benefit of module completer in ipy_completers.py
2823 2823 del self.shell.db['rootmodules']
2824 2824
2825 2825 path = [os.path.abspath(os.path.expanduser(p)) for p in
2826 2826 os.environ.get('PATH','').split(os.pathsep)]
2827 2827 path = filter(os.path.isdir,path)
2828 2828
2829 2829 syscmdlist = []
2830 2830 # Now define isexec in a cross platform manner.
2831 2831 if os.name == 'posix':
2832 2832 isexec = lambda fname:os.path.isfile(fname) and \
2833 2833 os.access(fname,os.X_OK)
2834 2834 else:
2835 2835 try:
2836 2836 winext = os.environ['pathext'].replace(';','|').replace('.','')
2837 2837 except KeyError:
2838 2838 winext = 'exe|com|bat|py'
2839 2839 if 'py' not in winext:
2840 2840 winext += '|py'
2841 2841 execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
2842 2842 isexec = lambda fname:os.path.isfile(fname) and execre.match(fname)
2843 2843 savedir = os.getcwdu()
2844 2844
2845 2845 # Now walk the paths looking for executables to alias.
2846 2846 try:
2847 2847 # write the whole loop for posix/Windows so we don't have an if in
2848 2848 # the innermost part
2849 2849 if os.name == 'posix':
2850 2850 for pdir in path:
2851 2851 os.chdir(pdir)
2852 2852 for ff in os.listdir(pdir):
2853 2853 if isexec(ff):
2854 2854 try:
2855 2855 # Removes dots from the name since ipython
2856 2856 # will assume names with dots to be python.
2857 2857 self.shell.alias_manager.define_alias(
2858 2858 ff.replace('.',''), ff)
2859 2859 except InvalidAliasError:
2860 2860 pass
2861 2861 else:
2862 2862 syscmdlist.append(ff)
2863 2863 else:
2864 2864 no_alias = self.shell.alias_manager.no_alias
2865 2865 for pdir in path:
2866 2866 os.chdir(pdir)
2867 2867 for ff in os.listdir(pdir):
2868 2868 base, ext = os.path.splitext(ff)
2869 2869 if isexec(ff) and base.lower() not in no_alias:
2870 2870 if ext.lower() == '.exe':
2871 2871 ff = base
2872 2872 try:
2873 2873 # Removes dots from the name since ipython
2874 2874 # will assume names with dots to be python.
2875 2875 self.shell.alias_manager.define_alias(
2876 2876 base.lower().replace('.',''), ff)
2877 2877 except InvalidAliasError:
2878 2878 pass
2879 2879 syscmdlist.append(ff)
2880 2880 self.shell.db['syscmdlist'] = syscmdlist
2881 2881 finally:
2882 2882 os.chdir(savedir)
2883 2883
2884 2884 @skip_doctest
2885 2885 def magic_pwd(self, parameter_s = ''):
2886 2886 """Return the current working directory path.
2887 2887
2888 2888 Examples
2889 2889 --------
2890 2890 ::
2891 2891
2892 2892 In [9]: pwd
2893 2893 Out[9]: '/home/tsuser/sprint/ipython'
2894 2894 """
2895 2895 return os.getcwdu()
2896 2896
2897 2897 @skip_doctest
2898 2898 def magic_cd(self, parameter_s=''):
2899 2899 """Change the current working directory.
2900 2900
2901 2901 This command automatically maintains an internal list of directories
2902 2902 you visit during your IPython session, in the variable _dh. The
2903 2903 command %dhist shows this history nicely formatted. You can also
2904 2904 do 'cd -<tab>' to see directory history conveniently.
2905 2905
2906 2906 Usage:
2907 2907
2908 2908 cd 'dir': changes to directory 'dir'.
2909 2909
2910 2910 cd -: changes to the last visited directory.
2911 2911
2912 2912 cd -<n>: changes to the n-th directory in the directory history.
2913 2913
2914 2914 cd --foo: change to directory that matches 'foo' in history
2915 2915
2916 2916 cd -b <bookmark_name>: jump to a bookmark set by %bookmark
2917 2917 (note: cd <bookmark_name> is enough if there is no
2918 2918 directory <bookmark_name>, but a bookmark with the name exists.)
2919 2919 'cd -b <tab>' allows you to tab-complete bookmark names.
2920 2920
2921 2921 Options:
2922 2922
2923 2923 -q: quiet. Do not print the working directory after the cd command is
2924 2924 executed. By default IPython's cd command does print this directory,
2925 2925 since the default prompts do not display path information.
2926 2926
2927 2927 Note that !cd doesn't work for this purpose because the shell where
2928 2928 !command runs is immediately discarded after executing 'command'.
2929 2929
2930 2930 Examples
2931 2931 --------
2932 2932 ::
2933 2933
2934 2934 In [10]: cd parent/child
2935 2935 /home/tsuser/parent/child
2936 2936 """
2937 2937
2938 2938 parameter_s = parameter_s.strip()
2939 2939 #bkms = self.shell.persist.get("bookmarks",{})
2940 2940
2941 2941 oldcwd = os.getcwdu()
2942 2942 numcd = re.match(r'(-)(\d+)$',parameter_s)
2943 2943 # jump in directory history by number
2944 2944 if numcd:
2945 2945 nn = int(numcd.group(2))
2946 2946 try:
2947 2947 ps = self.shell.user_ns['_dh'][nn]
2948 2948 except IndexError:
2949 2949 print 'The requested directory does not exist in history.'
2950 2950 return
2951 2951 else:
2952 2952 opts = {}
2953 2953 elif parameter_s.startswith('--'):
2954 2954 ps = None
2955 2955 fallback = None
2956 2956 pat = parameter_s[2:]
2957 2957 dh = self.shell.user_ns['_dh']
2958 2958 # first search only by basename (last component)
2959 2959 for ent in reversed(dh):
2960 2960 if pat in os.path.basename(ent) and os.path.isdir(ent):
2961 2961 ps = ent
2962 2962 break
2963 2963
2964 2964 if fallback is None and pat in ent and os.path.isdir(ent):
2965 2965 fallback = ent
2966 2966
2967 2967 # if we have no last part match, pick the first full path match
2968 2968 if ps is None:
2969 2969 ps = fallback
2970 2970
2971 2971 if ps is None:
2972 2972 print "No matching entry in directory history"
2973 2973 return
2974 2974 else:
2975 2975 opts = {}
2976 2976
2977 2977
2978 2978 else:
2979 2979 #turn all non-space-escaping backslashes to slashes,
2980 2980 # for c:\windows\directory\names\
2981 2981 parameter_s = re.sub(r'\\(?! )','/', parameter_s)
2982 2982 opts,ps = self.parse_options(parameter_s,'qb',mode='string')
2983 2983 # jump to previous
2984 2984 if ps == '-':
2985 2985 try:
2986 2986 ps = self.shell.user_ns['_dh'][-2]
2987 2987 except IndexError:
2988 2988 raise UsageError('%cd -: No previous directory to change to.')
2989 2989 # jump to bookmark if needed
2990 2990 else:
2991 2991 if not os.path.isdir(ps) or opts.has_key('b'):
2992 2992 bkms = self.shell.db.get('bookmarks', {})
2993 2993
2994 2994 if bkms.has_key(ps):
2995 2995 target = bkms[ps]
2996 2996 print '(bookmark:%s) -> %s' % (ps,target)
2997 2997 ps = target
2998 2998 else:
2999 2999 if opts.has_key('b'):
3000 3000 raise UsageError("Bookmark '%s' not found. "
3001 3001 "Use '%%bookmark -l' to see your bookmarks." % ps)
3002 3002
3003 3003 # strip extra quotes on Windows, because os.chdir doesn't like them
3004 3004 ps = unquote_filename(ps)
3005 3005 # at this point ps should point to the target dir
3006 3006 if ps:
3007 3007 try:
3008 3008 os.chdir(os.path.expanduser(ps))
3009 3009 if hasattr(self.shell, 'term_title') and self.shell.term_title:
3010 3010 set_term_title('IPython: ' + abbrev_cwd())
3011 3011 except OSError:
3012 3012 print sys.exc_info()[1]
3013 3013 else:
3014 3014 cwd = os.getcwdu()
3015 3015 dhist = self.shell.user_ns['_dh']
3016 3016 if oldcwd != cwd:
3017 3017 dhist.append(cwd)
3018 3018 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
3019 3019
3020 3020 else:
3021 3021 os.chdir(self.shell.home_dir)
3022 3022 if hasattr(self.shell, 'term_title') and self.shell.term_title:
3023 3023 set_term_title('IPython: ' + '~')
3024 3024 cwd = os.getcwdu()
3025 3025 dhist = self.shell.user_ns['_dh']
3026 3026
3027 3027 if oldcwd != cwd:
3028 3028 dhist.append(cwd)
3029 3029 self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
3030 3030 if not 'q' in opts and self.shell.user_ns['_dh']:
3031 3031 print self.shell.user_ns['_dh'][-1]
3032 3032
3033 3033
3034 3034 def magic_env(self, parameter_s=''):
3035 3035 """List environment variables."""
3036 3036
3037 3037 return dict(os.environ)
3038 3038
3039 3039 def magic_pushd(self, parameter_s=''):
3040 3040 """Place the current dir on stack and change directory.
3041 3041
3042 3042 Usage:\\
3043 3043 %pushd ['dirname']
3044 3044 """
3045 3045
3046 3046 dir_s = self.shell.dir_stack
3047 3047 tgt = os.path.expanduser(unquote_filename(parameter_s))
3048 cwd = os.getcwdu().replace(self.home_dir,'~')
3048 cwd = os.getcwdu().replace(self.shell.home_dir,'~')
3049 3049 if tgt:
3050 3050 self.magic_cd(parameter_s)
3051 3051 dir_s.insert(0,cwd)
3052 return self.magic_dirs()
3052 return self.shell.magic('dirs')
3053 3053
3054 3054 def magic_popd(self, parameter_s=''):
3055 3055 """Change to directory popped off the top of the stack.
3056 3056 """
3057 3057 if not self.shell.dir_stack:
3058 3058 raise UsageError("%popd on empty stack")
3059 3059 top = self.shell.dir_stack.pop(0)
3060 3060 self.magic_cd(top)
3061 3061 print "popd ->",top
3062 3062
3063 3063 def magic_dirs(self, parameter_s=''):
3064 3064 """Return the current directory stack."""
3065 3065
3066 3066 return self.shell.dir_stack
3067 3067
3068 3068 def magic_dhist(self, parameter_s=''):
3069 3069 """Print your history of visited directories.
3070 3070
3071 3071 %dhist -> print full history\\
3072 3072 %dhist n -> print last n entries only\\
3073 3073 %dhist n1 n2 -> print entries between n1 and n2 (n1 not included)\\
3074 3074
3075 3075 This history is automatically maintained by the %cd command, and
3076 3076 always available as the global list variable _dh. You can use %cd -<n>
3077 3077 to go to directory number <n>.
3078 3078
3079 3079 Note that most of time, you should view directory history by entering
3080 3080 cd -<TAB>.
3081 3081
3082 3082 """
3083 3083
3084 3084 dh = self.shell.user_ns['_dh']
3085 3085 if parameter_s:
3086 3086 try:
3087 3087 args = map(int,parameter_s.split())
3088 3088 except:
3089 3089 self.arg_err(Magic.magic_dhist)
3090 3090 return
3091 3091 if len(args) == 1:
3092 3092 ini,fin = max(len(dh)-(args[0]),0),len(dh)
3093 3093 elif len(args) == 2:
3094 3094 ini,fin = args
3095 3095 else:
3096 3096 self.arg_err(Magic.magic_dhist)
3097 3097 return
3098 3098 else:
3099 3099 ini,fin = 0,len(dh)
3100 3100 nlprint(dh,
3101 3101 header = 'Directory history (kept in _dh)',
3102 3102 start=ini,stop=fin)
3103 3103
3104 3104 @skip_doctest
3105 3105 def magic_sc(self, parameter_s=''):
3106 3106 """Shell capture - execute a shell command and capture its output.
3107 3107
3108 3108 DEPRECATED. Suboptimal, retained for backwards compatibility.
3109 3109
3110 3110 You should use the form 'var = !command' instead. Example:
3111 3111
3112 3112 "%sc -l myfiles = ls ~" should now be written as
3113 3113
3114 3114 "myfiles = !ls ~"
3115 3115
3116 3116 myfiles.s, myfiles.l and myfiles.n still apply as documented
3117 3117 below.
3118 3118
3119 3119 --
3120 3120 %sc [options] varname=command
3121 3121
3122 3122 IPython will run the given command using commands.getoutput(), and
3123 3123 will then update the user's interactive namespace with a variable
3124 3124 called varname, containing the value of the call. Your command can
3125 3125 contain shell wildcards, pipes, etc.
3126 3126
3127 3127 The '=' sign in the syntax is mandatory, and the variable name you
3128 3128 supply must follow Python's standard conventions for valid names.
3129 3129
3130 3130 (A special format without variable name exists for internal use)
3131 3131
3132 3132 Options:
3133 3133
3134 3134 -l: list output. Split the output on newlines into a list before
3135 3135 assigning it to the given variable. By default the output is stored
3136 3136 as a single string.
3137 3137
3138 3138 -v: verbose. Print the contents of the variable.
3139 3139
3140 3140 In most cases you should not need to split as a list, because the
3141 3141 returned value is a special type of string which can automatically
3142 3142 provide its contents either as a list (split on newlines) or as a
3143 3143 space-separated string. These are convenient, respectively, either
3144 3144 for sequential processing or to be passed to a shell command.
3145 3145
3146 3146 For example::
3147 3147
3148 3148 # Capture into variable a
3149 3149 In [1]: sc a=ls *py
3150 3150
3151 3151 # a is a string with embedded newlines
3152 3152 In [2]: a
3153 3153 Out[2]: 'setup.py\\nwin32_manual_post_install.py'
3154 3154
3155 3155 # which can be seen as a list:
3156 3156 In [3]: a.l
3157 3157 Out[3]: ['setup.py', 'win32_manual_post_install.py']
3158 3158
3159 3159 # or as a whitespace-separated string:
3160 3160 In [4]: a.s
3161 3161 Out[4]: 'setup.py win32_manual_post_install.py'
3162 3162
3163 3163 # a.s is useful to pass as a single command line:
3164 3164 In [5]: !wc -l $a.s
3165 3165 146 setup.py
3166 3166 130 win32_manual_post_install.py
3167 3167 276 total
3168 3168
3169 3169 # while the list form is useful to loop over:
3170 3170 In [6]: for f in a.l:
3171 3171 ...: !wc -l $f
3172 3172 ...:
3173 3173 146 setup.py
3174 3174 130 win32_manual_post_install.py
3175 3175
3176 3176 Similarly, the lists returned by the -l option are also special, in
3177 3177 the sense that you can equally invoke the .s attribute on them to
3178 3178 automatically get a whitespace-separated string from their contents::
3179 3179
3180 3180 In [7]: sc -l b=ls *py
3181 3181
3182 3182 In [8]: b
3183 3183 Out[8]: ['setup.py', 'win32_manual_post_install.py']
3184 3184
3185 3185 In [9]: b.s
3186 3186 Out[9]: 'setup.py win32_manual_post_install.py'
3187 3187
3188 3188 In summary, both the lists and strings used for output capture have
3189 3189 the following special attributes::
3190 3190
3191 3191 .l (or .list) : value as list.
3192 3192 .n (or .nlstr): value as newline-separated string.
3193 3193 .s (or .spstr): value as space-separated string.
3194 3194 """
3195 3195
3196 3196 opts,args = self.parse_options(parameter_s,'lv')
3197 3197 # Try to get a variable name and command to run
3198 3198 try:
3199 3199 # the variable name must be obtained from the parse_options
3200 3200 # output, which uses shlex.split to strip options out.
3201 3201 var,_ = args.split('=',1)
3202 3202 var = var.strip()
3203 3203 # But the command has to be extracted from the original input
3204 3204 # parameter_s, not on what parse_options returns, to avoid the
3205 3205 # quote stripping which shlex.split performs on it.
3206 3206 _,cmd = parameter_s.split('=',1)
3207 3207 except ValueError:
3208 3208 var,cmd = '',''
3209 3209 # If all looks ok, proceed
3210 3210 split = 'l' in opts
3211 3211 out = self.shell.getoutput(cmd, split=split)
3212 3212 if opts.has_key('v'):
3213 3213 print '%s ==\n%s' % (var,pformat(out))
3214 3214 if var:
3215 3215 self.shell.user_ns.update({var:out})
3216 3216 else:
3217 3217 return out
3218 3218
3219 3219 def magic_sx(self, parameter_s=''):
3220 3220 """Shell execute - run a shell command and capture its output.
3221 3221
3222 3222 %sx command
3223 3223
3224 3224 IPython will run the given command using commands.getoutput(), and
3225 3225 return the result formatted as a list (split on '\\n'). Since the
3226 3226 output is _returned_, it will be stored in ipython's regular output
3227 3227 cache Out[N] and in the '_N' automatic variables.
3228 3228
3229 3229 Notes:
3230 3230
3231 3231 1) If an input line begins with '!!', then %sx is automatically
3232 3232 invoked. That is, while::
3233 3233
3234 3234 !ls
3235 3235
3236 3236 causes ipython to simply issue system('ls'), typing::
3237 3237
3238 3238 !!ls
3239 3239
3240 3240 is a shorthand equivalent to::
3241 3241
3242 3242 %sx ls
3243 3243
3244 3244 2) %sx differs from %sc in that %sx automatically splits into a list,
3245 3245 like '%sc -l'. The reason for this is to make it as easy as possible
3246 3246 to process line-oriented shell output via further python commands.
3247 3247 %sc is meant to provide much finer control, but requires more
3248 3248 typing.
3249 3249
3250 3250 3) Just like %sc -l, this is a list with special attributes:
3251 3251 ::
3252 3252
3253 3253 .l (or .list) : value as list.
3254 3254 .n (or .nlstr): value as newline-separated string.
3255 3255 .s (or .spstr): value as whitespace-separated string.
3256 3256
3257 3257 This is very useful when trying to use such lists as arguments to
3258 3258 system commands."""
3259 3259
3260 3260 if parameter_s:
3261 3261 return self.shell.getoutput(parameter_s)
3262 3262
3263 3263
3264 3264 def magic_bookmark(self, parameter_s=''):
3265 3265 """Manage IPython's bookmark system.
3266 3266
3267 3267 %bookmark <name> - set bookmark to current dir
3268 3268 %bookmark <name> <dir> - set bookmark to <dir>
3269 3269 %bookmark -l - list all bookmarks
3270 3270 %bookmark -d <name> - remove bookmark
3271 3271 %bookmark -r - remove all bookmarks
3272 3272
3273 3273 You can later on access a bookmarked folder with::
3274 3274
3275 3275 %cd -b <name>
3276 3276
3277 3277 or simply '%cd <name>' if there is no directory called <name> AND
3278 3278 there is such a bookmark defined.
3279 3279
3280 3280 Your bookmarks persist through IPython sessions, but they are
3281 3281 associated with each profile."""
3282 3282
3283 3283 opts,args = self.parse_options(parameter_s,'drl',mode='list')
3284 3284 if len(args) > 2:
3285 3285 raise UsageError("%bookmark: too many arguments")
3286 3286
3287 3287 bkms = self.shell.db.get('bookmarks',{})
3288 3288
3289 3289 if opts.has_key('d'):
3290 3290 try:
3291 3291 todel = args[0]
3292 3292 except IndexError:
3293 3293 raise UsageError(
3294 3294 "%bookmark -d: must provide a bookmark to delete")
3295 3295 else:
3296 3296 try:
3297 3297 del bkms[todel]
3298 3298 except KeyError:
3299 3299 raise UsageError(
3300 3300 "%%bookmark -d: Can't delete bookmark '%s'" % todel)
3301 3301
3302 3302 elif opts.has_key('r'):
3303 3303 bkms = {}
3304 3304 elif opts.has_key('l'):
3305 3305 bks = bkms.keys()
3306 3306 bks.sort()
3307 3307 if bks:
3308 3308 size = max(map(len,bks))
3309 3309 else:
3310 3310 size = 0
3311 3311 fmt = '%-'+str(size)+'s -> %s'
3312 3312 print 'Current bookmarks:'
3313 3313 for bk in bks:
3314 3314 print fmt % (bk,bkms[bk])
3315 3315 else:
3316 3316 if not args:
3317 3317 raise UsageError("%bookmark: You must specify the bookmark name")
3318 3318 elif len(args)==1:
3319 3319 bkms[args[0]] = os.getcwdu()
3320 3320 elif len(args)==2:
3321 3321 bkms[args[0]] = args[1]
3322 3322 self.shell.db['bookmarks'] = bkms
3323 3323
3324 3324
3325 3325 def magic_pycat(self, parameter_s=''):
3326 3326 """Show a syntax-highlighted file through a pager.
3327 3327
3328 3328 This magic is similar to the cat utility, but it will assume the file
3329 3329 to be Python source and will show it with syntax highlighting.
3330 3330
3331 3331 This magic command can either take a local filename, an url,
3332 3332 an history range (see %history) or a macro as argument ::
3333 3333
3334 3334 %pycat myscript.py
3335 3335 %pycat 7-27
3336 3336 %pycat myMacro
3337 3337 %pycat http://www.example.com/myscript.py
3338 3338 """
3339 3339
3340 3340 try :
3341 3341 cont = self.shell.find_user_code(parameter_s)
3342 3342 except ValueError, IOError:
3343 3343 print "Error: no such file, variable, URL, history range or macro"
3344 3344 return
3345 3345
3346 3346 page.page(self.shell.pycolorize(cont))
3347 3347
3348 3348 def magic_quickref(self,arg):
3349 3349 """ Show a quick reference sheet """
3350 3350 import IPython.core.usage
3351 3351 qr = IPython.core.usage.quick_reference + self.magic_magic('-brief')
3352 3352
3353 3353 page.page(qr)
3354 3354
3355 3355 def magic_doctest_mode(self,parameter_s=''):
3356 3356 """Toggle doctest mode on and off.
3357 3357
3358 3358 This mode is intended to make IPython behave as much as possible like a
3359 3359 plain Python shell, from the perspective of how its prompts, exceptions
3360 3360 and output look. This makes it easy to copy and paste parts of a
3361 3361 session into doctests. It does so by:
3362 3362
3363 3363 - Changing the prompts to the classic ``>>>`` ones.
3364 3364 - Changing the exception reporting mode to 'Plain'.
3365 3365 - Disabling pretty-printing of output.
3366 3366
3367 3367 Note that IPython also supports the pasting of code snippets that have
3368 3368 leading '>>>' and '...' prompts in them. This means that you can paste
3369 3369 doctests from files or docstrings (even if they have leading
3370 3370 whitespace), and the code will execute correctly. You can then use
3371 3371 '%history -t' to see the translated history; this will give you the
3372 3372 input after removal of all the leading prompts and whitespace, which
3373 3373 can be pasted back into an editor.
3374 3374
3375 3375 With these features, you can switch into this mode easily whenever you
3376 3376 need to do testing and changes to doctests, without having to leave
3377 3377 your existing IPython session.
3378 3378 """
3379 3379
3380 3380 from IPython.utils.ipstruct import Struct
3381 3381
3382 3382 # Shorthands
3383 3383 shell = self.shell
3384 3384 pm = shell.prompt_manager
3385 3385 meta = shell.meta
3386 3386 disp_formatter = self.shell.display_formatter
3387 3387 ptformatter = disp_formatter.formatters['text/plain']
3388 3388 # dstore is a data store kept in the instance metadata bag to track any
3389 3389 # changes we make, so we can undo them later.
3390 3390 dstore = meta.setdefault('doctest_mode',Struct())
3391 3391 save_dstore = dstore.setdefault
3392 3392
3393 3393 # save a few values we'll need to recover later
3394 3394 mode = save_dstore('mode',False)
3395 3395 save_dstore('rc_pprint',ptformatter.pprint)
3396 3396 save_dstore('xmode',shell.InteractiveTB.mode)
3397 3397 save_dstore('rc_separate_out',shell.separate_out)
3398 3398 save_dstore('rc_separate_out2',shell.separate_out2)
3399 3399 save_dstore('rc_prompts_pad_left',pm.justify)
3400 3400 save_dstore('rc_separate_in',shell.separate_in)
3401 3401 save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
3402 3402 save_dstore('prompt_templates',(pm.in_template, pm.in2_template, pm.out_template))
3403 3403
3404 3404 if mode == False:
3405 3405 # turn on
3406 3406 pm.in_template = '>>> '
3407 3407 pm.in2_template = '... '
3408 3408 pm.out_template = ''
3409 3409
3410 3410 # Prompt separators like plain python
3411 3411 shell.separate_in = ''
3412 3412 shell.separate_out = ''
3413 3413 shell.separate_out2 = ''
3414 3414
3415 3415 pm.justify = False
3416 3416
3417 3417 ptformatter.pprint = False
3418 3418 disp_formatter.plain_text_only = True
3419 3419
3420 shell.magic_xmode('Plain')
3420 shell.magic('xmode Plain')
3421 3421 else:
3422 3422 # turn off
3423 3423 pm.in_template, pm.in2_template, pm.out_template = dstore.prompt_templates
3424 3424
3425 3425 shell.separate_in = dstore.rc_separate_in
3426 3426
3427 3427 shell.separate_out = dstore.rc_separate_out
3428 3428 shell.separate_out2 = dstore.rc_separate_out2
3429 3429
3430 3430 pm.justify = dstore.rc_prompts_pad_left
3431 3431
3432 3432 ptformatter.pprint = dstore.rc_pprint
3433 3433 disp_formatter.plain_text_only = dstore.rc_plain_text_only
3434 3434
3435 shell.magic_xmode(dstore.xmode)
3435 shell.magic('xmode ' + dstore.xmode)
3436 3436
3437 3437 # Store new mode and inform
3438 3438 dstore.mode = bool(1-int(mode))
3439 3439 mode_label = ['OFF','ON'][dstore.mode]
3440 3440 print 'Doctest mode is:', mode_label
3441 3441
3442 3442 def magic_gui(self, parameter_s=''):
3443 3443 """Enable or disable IPython GUI event loop integration.
3444 3444
3445 3445 %gui [GUINAME]
3446 3446
3447 3447 This magic replaces IPython's threaded shells that were activated
3448 3448 using the (pylab/wthread/etc.) command line flags. GUI toolkits
3449 3449 can now be enabled at runtime and keyboard
3450 3450 interrupts should work without any problems. The following toolkits
3451 3451 are supported: wxPython, PyQt4, PyGTK, Tk and Cocoa (OSX)::
3452 3452
3453 3453 %gui wx # enable wxPython event loop integration
3454 3454 %gui qt4|qt # enable PyQt4 event loop integration
3455 3455 %gui gtk # enable PyGTK event loop integration
3456 3456 %gui gtk3 # enable Gtk3 event loop integration
3457 3457 %gui tk # enable Tk event loop integration
3458 3458 %gui OSX # enable Cocoa event loop integration
3459 3459 # (requires %matplotlib 1.1)
3460 3460 %gui # disable all event loop integration
3461 3461
3462 3462 WARNING: after any of these has been called you can simply create
3463 3463 an application object, but DO NOT start the event loop yourself, as
3464 3464 we have already handled that.
3465 3465 """
3466 3466 opts, arg = self.parse_options(parameter_s, '')
3467 3467 if arg=='': arg = None
3468 3468 try:
3469 3469 return self.enable_gui(arg)
3470 3470 except Exception as e:
3471 3471 # print simple error message, rather than traceback if we can't
3472 3472 # hook up the GUI
3473 3473 error(str(e))
3474 3474
3475 3475 def magic_install_ext(self, parameter_s):
3476 3476 """Download and install an extension from a URL, e.g.::
3477 3477
3478 3478 %install_ext https://bitbucket.org/birkenfeld/ipython-physics/raw/d1310a2ab15d/physics.py
3479 3479
3480 3480 The URL should point to an importable Python module - either a .py file
3481 3481 or a .zip file.
3482 3482
3483 3483 Parameters:
3484 3484
3485 3485 -n filename : Specify a name for the file, rather than taking it from
3486 3486 the URL.
3487 3487 """
3488 3488 opts, args = self.parse_options(parameter_s, 'n:')
3489 3489 try:
3490 filename = self.extension_manager.install_extension(args, opts.get('n'))
3490 filename = self.shell.extension_manager.install_extension(args,
3491 opts.get('n'))
3491 3492 except ValueError as e:
3492 3493 print e
3493 3494 return
3494 3495
3495 3496 filename = os.path.basename(filename)
3496 3497 print "Installed %s. To use it, type:" % filename
3497 3498 print " %%load_ext %s" % os.path.splitext(filename)[0]
3498 3499
3499 3500
3500 3501 def magic_load_ext(self, module_str):
3501 3502 """Load an IPython extension by its module name."""
3502 return self.extension_manager.load_extension(module_str)
3503 return self.shell.extension_manager.load_extension(module_str)
3503 3504
3504 3505 def magic_unload_ext(self, module_str):
3505 3506 """Unload an IPython extension by its module name."""
3506 self.extension_manager.unload_extension(module_str)
3507 self.shell.extension_manager.unload_extension(module_str)
3507 3508
3508 3509 def magic_reload_ext(self, module_str):
3509 3510 """Reload an IPython extension by its module name."""
3510 self.extension_manager.reload_extension(module_str)
3511 self.shell.extension_manager.reload_extension(module_str)
3511 3512
3512 3513 def magic_install_profiles(self, s):
3513 3514 """%install_profiles has been deprecated."""
3514 3515 print '\n'.join([
3515 3516 "%install_profiles has been deprecated.",
3516 3517 "Use `ipython profile list` to view available profiles.",
3517 3518 "Requesting a profile with `ipython profile create <name>`",
3518 3519 "or `ipython --profile=<name>` will start with the bundled",
3519 3520 "profile of that name if it exists."
3520 3521 ])
3521 3522
3522 3523 def magic_install_default_config(self, s):
3523 3524 """%install_default_config has been deprecated."""
3524 3525 print '\n'.join([
3525 3526 "%install_default_config has been deprecated.",
3526 3527 "Use `ipython profile create <name>` to initialize a profile",
3527 3528 "with the default config files.",
3528 3529 "Add `--reset` to overwrite already existing config files with defaults."
3529 3530 ])
3530 3531
3531 3532 @skip_doctest
3532 3533 def magic_pylab(self, s):
3533 3534 """Load numpy and matplotlib to work interactively.
3534 3535
3535 3536 %pylab [GUINAME]
3536 3537
3537 3538 This function lets you activate pylab (matplotlib, numpy and
3538 3539 interactive support) at any point during an IPython session.
3539 3540
3540 3541 It will import at the top level numpy as np, pyplot as plt, matplotlib,
3541 3542 pylab and mlab, as well as all names from numpy and pylab.
3542 3543
3543 3544 If you are using the inline matplotlib backend for embedded figures,
3544 3545 you can adjust its behavior via the %config magic::
3545 3546
3546 3547 # enable SVG figures, necessary for SVG+XHTML export in the qtconsole
3547 3548 In [1]: %config InlineBackend.figure_format = 'svg'
3548 3549
3549 3550 # change the behavior of closing all figures at the end of each
3550 3551 # execution (cell), or allowing reuse of active figures across
3551 3552 # cells:
3552 3553 In [2]: %config InlineBackend.close_figures = False
3553 3554
3554 3555 Parameters
3555 3556 ----------
3556 3557 guiname : optional
3557 3558 One of the valid arguments to the %gui magic ('qt', 'wx', 'gtk',
3558 3559 'osx' or 'tk'). If given, the corresponding Matplotlib backend is
3559 3560 used, otherwise matplotlib's default (which you can override in your
3560 3561 matplotlib config file) is used.
3561 3562
3562 3563 Examples
3563 3564 --------
3564 3565 In this case, where the MPL default is TkAgg::
3565 3566
3566 3567 In [2]: %pylab
3567 3568
3568 3569 Welcome to pylab, a matplotlib-based Python environment.
3569 3570 Backend in use: TkAgg
3570 3571 For more information, type 'help(pylab)'.
3571 3572
3572 3573 But you can explicitly request a different backend::
3573 3574
3574 3575 In [3]: %pylab qt
3575 3576
3576 3577 Welcome to pylab, a matplotlib-based Python environment.
3577 3578 Backend in use: Qt4Agg
3578 3579 For more information, type 'help(pylab)'.
3579 3580 """
3580 3581
3581 3582 if Application.initialized():
3582 3583 app = Application.instance()
3583 3584 try:
3584 3585 import_all_status = app.pylab_import_all
3585 3586 except AttributeError:
3586 3587 import_all_status = True
3587 3588 else:
3588 3589 import_all_status = True
3589 3590
3590 3591 self.shell.enable_pylab(s, import_all=import_all_status)
3591 3592
3592 3593 def magic_tb(self, s):
3593 3594 """Print the last traceback with the currently active exception mode.
3594 3595
3595 3596 See %xmode for changing exception reporting modes."""
3596 3597 self.shell.showtraceback()
3597 3598
3598 3599 @skip_doctest
3599 3600 def magic_precision(self, s=''):
3600 3601 """Set floating point precision for pretty printing.
3601 3602
3602 3603 Can set either integer precision or a format string.
3603 3604
3604 3605 If numpy has been imported and precision is an int,
3605 3606 numpy display precision will also be set, via ``numpy.set_printoptions``.
3606 3607
3607 3608 If no argument is given, defaults will be restored.
3608 3609
3609 3610 Examples
3610 3611 --------
3611 3612 ::
3612 3613
3613 3614 In [1]: from math import pi
3614 3615
3615 3616 In [2]: %precision 3
3616 3617 Out[2]: u'%.3f'
3617 3618
3618 3619 In [3]: pi
3619 3620 Out[3]: 3.142
3620 3621
3621 3622 In [4]: %precision %i
3622 3623 Out[4]: u'%i'
3623 3624
3624 3625 In [5]: pi
3625 3626 Out[5]: 3
3626 3627
3627 3628 In [6]: %precision %e
3628 3629 Out[6]: u'%e'
3629 3630
3630 3631 In [7]: pi**10
3631 3632 Out[7]: 9.364805e+04
3632 3633
3633 3634 In [8]: %precision
3634 3635 Out[8]: u'%r'
3635 3636
3636 3637 In [9]: pi**10
3637 3638 Out[9]: 93648.047476082982
3638 3639
3639 3640 """
3640 3641
3641 3642 ptformatter = self.shell.display_formatter.formatters['text/plain']
3642 3643 ptformatter.float_precision = s
3643 3644 return ptformatter.float_format
3644 3645
3645 3646
3646 3647 @magic_arguments.magic_arguments()
3647 3648 @magic_arguments.argument(
3648 3649 '-e', '--export', action='store_true', default=False,
3649 3650 help='Export IPython history as a notebook. The filename argument '
3650 3651 'is used to specify the notebook name and format. For example '
3651 3652 'a filename of notebook.ipynb will result in a notebook name '
3652 3653 'of "notebook" and a format of "xml". Likewise using a ".json" '
3653 3654 'or ".py" file extension will write the notebook in the json '
3654 3655 'or py formats.'
3655 3656 )
3656 3657 @magic_arguments.argument(
3657 3658 '-f', '--format',
3658 3659 help='Convert an existing IPython notebook to a new format. This option '
3659 3660 'specifies the new format and can have the values: xml, json, py. '
3660 3661 'The target filename is chosen automatically based on the new '
3661 3662 'format. The filename argument gives the name of the source file.'
3662 3663 )
3663 3664 @magic_arguments.argument(
3664 3665 'filename', type=unicode,
3665 3666 help='Notebook name or filename'
3666 3667 )
3667 3668 def magic_notebook(self, s):
3668 3669 """Export and convert IPython notebooks.
3669 3670
3670 3671 This function can export the current IPython history to a notebook file
3671 3672 or can convert an existing notebook file into a different format. For
3672 3673 example, to export the history to "foo.ipynb" do "%notebook -e foo.ipynb".
3673 3674 To export the history to "foo.py" do "%notebook -e foo.py". To convert
3674 3675 "foo.ipynb" to "foo.json" do "%notebook -f json foo.ipynb". Possible
3675 3676 formats include (json/ipynb, py).
3676 3677 """
3677 3678 args = magic_arguments.parse_argstring(self.magic_notebook, s)
3678 3679
3679 3680 from IPython.nbformat import current
3680 3681 args.filename = unquote_filename(args.filename)
3681 3682 if args.export:
3682 3683 fname, name, format = current.parse_filename(args.filename)
3683 3684 cells = []
3684 hist = list(self.history_manager.get_range())
3685 hist = list(self.shell.history_manager.get_range())
3685 3686 for session, prompt_number, input in hist[:-1]:
3686 cells.append(current.new_code_cell(prompt_number=prompt_number, input=input))
3687 cells.append(current.new_code_cell(prompt_number=prompt_number,
3688 input=input))
3687 3689 worksheet = current.new_worksheet(cells=cells)
3688 3690 nb = current.new_notebook(name=name,worksheets=[worksheet])
3689 3691 with io.open(fname, 'w', encoding='utf-8') as f:
3690 3692 current.write(nb, f, format);
3691 3693 elif args.format is not None:
3692 3694 old_fname, old_name, old_format = current.parse_filename(args.filename)
3693 3695 new_format = args.format
3694 3696 if new_format == u'xml':
3695 3697 raise ValueError('Notebooks cannot be written as xml.')
3696 3698 elif new_format == u'ipynb' or new_format == u'json':
3697 3699 new_fname = old_name + u'.ipynb'
3698 3700 new_format = u'json'
3699 3701 elif new_format == u'py':
3700 3702 new_fname = old_name + u'.py'
3701 3703 else:
3702 3704 raise ValueError('Invalid notebook format: %s' % new_format)
3703 3705 with io.open(old_fname, 'r', encoding='utf-8') as f:
3704 3706 nb = current.read(f, old_format)
3705 3707 with io.open(new_fname, 'w', encoding='utf-8') as f:
3706 3708 current.write(nb, f, new_format)
3707 3709
3708 3710 def magic_config(self, s):
3709 3711 """configure IPython
3710 3712
3711 3713 %config Class[.trait=value]
3712 3714
3713 3715 This magic exposes most of the IPython config system. Any
3714 3716 Configurable class should be able to be configured with the simple
3715 3717 line::
3716 3718
3717 3719 %config Class.trait=value
3718 3720
3719 3721 Where `value` will be resolved in the user's namespace, if it is an
3720 3722 expression or variable name.
3721 3723
3722 3724 Examples
3723 3725 --------
3724 3726
3725 3727 To see what classes are available for config, pass no arguments::
3726 3728
3727 3729 In [1]: %config
3728 3730 Available objects for config:
3729 3731 TerminalInteractiveShell
3730 3732 HistoryManager
3731 3733 PrefilterManager
3732 3734 AliasManager
3733 3735 IPCompleter
3734 3736 PromptManager
3735 3737 DisplayFormatter
3736 3738
3737 3739 To view what is configurable on a given class, just pass the class
3738 3740 name::
3739 3741
3740 3742 In [2]: %config IPCompleter
3741 3743 IPCompleter options
3742 3744 -----------------
3743 3745 IPCompleter.omit__names=<Enum>
3744 3746 Current: 2
3745 3747 Choices: (0, 1, 2)
3746 3748 Instruct the completer to omit private method names
3747 3749 Specifically, when completing on ``object.<tab>``.
3748 3750 When 2 [default]: all names that start with '_' will be excluded.
3749 3751 When 1: all 'magic' names (``__foo__``) will be excluded.
3750 3752 When 0: nothing will be excluded.
3751 3753 IPCompleter.merge_completions=<CBool>
3752 3754 Current: True
3753 3755 Whether to merge completion results into a single list
3754 3756 If False, only the completion results from the first non-empty completer
3755 3757 will be returned.
3756 3758 IPCompleter.limit_to__all__=<CBool>
3757 3759 Current: False
3758 3760 Instruct the completer to use __all__ for the completion
3759 3761 Specifically, when completing on ``object.<tab>``.
3760 3762 When True: only those names in obj.__all__ will be included.
3761 3763 When False [default]: the __all__ attribute is ignored
3762 3764 IPCompleter.greedy=<CBool>
3763 3765 Current: False
3764 3766 Activate greedy completion
3765 3767 This will enable completion on elements of lists, results of function calls,
3766 3768 etc., but can be unsafe because the code is actually evaluated on TAB.
3767 3769
3768 3770 but the real use is in setting values::
3769 3771
3770 3772 In [3]: %config IPCompleter.greedy = True
3771 3773
3772 3774 and these values are read from the user_ns if they are variables::
3773 3775
3774 3776 In [4]: feeling_greedy=False
3775 3777
3776 3778 In [5]: %config IPCompleter.greedy = feeling_greedy
3777 3779
3778 3780 """
3779 3781 from IPython.config.loader import Config
3780 3782 # some IPython objects are Configurable, but do not yet have
3781 3783 # any configurable traits. Exclude them from the effects of
3782 3784 # this magic, as their presence is just noise:
3783 3785 configurables = [ c for c in self.shell.configurables
3784 3786 if c.__class__.class_traits(config=True) ]
3785 3787 classnames = [ c.__class__.__name__ for c in configurables ]
3786 3788
3787 3789 line = s.strip()
3788 3790 if not line:
3789 3791 # print available configurable names
3790 3792 print "Available objects for config:"
3791 3793 for name in classnames:
3792 3794 print " ", name
3793 3795 return
3794 3796 elif line in classnames:
3795 3797 # `%config TerminalInteractiveShell` will print trait info for
3796 3798 # TerminalInteractiveShell
3797 3799 c = configurables[classnames.index(line)]
3798 3800 cls = c.__class__
3799 3801 help = cls.class_get_help(c)
3800 3802 # strip leading '--' from cl-args:
3801 3803 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
3802 3804 print help
3803 3805 return
3804 3806 elif '=' not in line:
3805 3807 raise UsageError("Invalid config statement: %r, should be Class.trait = value" % line)
3806 3808
3807 3809
3808 3810 # otherwise, assume we are setting configurables.
3809 3811 # leave quotes on args when splitting, because we want
3810 3812 # unquoted args to eval in user_ns
3811 3813 cfg = Config()
3812 3814 exec "cfg."+line in locals(), self.shell.user_ns
3813 3815
3814 3816 for configurable in configurables:
3815 3817 try:
3816 3818 configurable.update_config(cfg)
3817 3819 except Exception as e:
3818 3820 error(e)
3819 3821
3820 3822 # end Magic
@@ -1,359 +1,359 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the key interactiveshell module.
3 3
4 4 Historically the main classes in interactiveshell have been under-tested. This
5 5 module should grow as many single-method tests as possible to trap many of the
6 6 recurring bugs we seem to encounter with high-level interaction.
7 7
8 8 Authors
9 9 -------
10 10 * Fernando Perez
11 11 """
12 12 #-----------------------------------------------------------------------------
13 13 # Copyright (C) 2011 The IPython Development Team
14 14 #
15 15 # Distributed under the terms of the BSD License. The full license is in
16 16 # the file COPYING, distributed as part of this software.
17 17 #-----------------------------------------------------------------------------
18 18
19 19 #-----------------------------------------------------------------------------
20 20 # Imports
21 21 #-----------------------------------------------------------------------------
22 22 # stdlib
23 23 import os
24 24 import shutil
25 25 import tempfile
26 26 import unittest
27 27 from os.path import join
28 28 import sys
29 29 from StringIO import StringIO
30 30
31 31 from IPython.testing.decorators import skipif
32 32 from IPython.utils import io
33 33
34 34 #-----------------------------------------------------------------------------
35 35 # Tests
36 36 #-----------------------------------------------------------------------------
37 37
38 38 class InteractiveShellTestCase(unittest.TestCase):
39 39 def test_naked_string_cells(self):
40 40 """Test that cells with only naked strings are fully executed"""
41 41 ip = get_ipython()
42 42 # First, single-line inputs
43 43 ip.run_cell('"a"\n')
44 44 self.assertEquals(ip.user_ns['_'], 'a')
45 45 # And also multi-line cells
46 46 ip.run_cell('"""a\nb"""\n')
47 47 self.assertEquals(ip.user_ns['_'], 'a\nb')
48 48
49 49 def test_run_empty_cell(self):
50 50 """Just make sure we don't get a horrible error with a blank
51 51 cell of input. Yes, I did overlook that."""
52 52 ip = get_ipython()
53 53 old_xc = ip.execution_count
54 54 ip.run_cell('')
55 55 self.assertEquals(ip.execution_count, old_xc)
56 56
57 57 def test_run_cell_multiline(self):
58 58 """Multi-block, multi-line cells must execute correctly.
59 59 """
60 60 ip = get_ipython()
61 61 src = '\n'.join(["x=1",
62 62 "y=2",
63 63 "if 1:",
64 64 " x += 1",
65 65 " y += 1",])
66 66 ip.run_cell(src)
67 67 self.assertEquals(ip.user_ns['x'], 2)
68 68 self.assertEquals(ip.user_ns['y'], 3)
69 69
70 70 def test_multiline_string_cells(self):
71 71 "Code sprinkled with multiline strings should execute (GH-306)"
72 72 ip = get_ipython()
73 73 ip.run_cell('tmp=0')
74 74 self.assertEquals(ip.user_ns['tmp'], 0)
75 75 ip.run_cell('tmp=1;"""a\nb"""\n')
76 76 self.assertEquals(ip.user_ns['tmp'], 1)
77 77
78 78 def test_dont_cache_with_semicolon(self):
79 79 "Ending a line with semicolon should not cache the returned object (GH-307)"
80 80 ip = get_ipython()
81 81 oldlen = len(ip.user_ns['Out'])
82 82 a = ip.run_cell('1;', store_history=True)
83 83 newlen = len(ip.user_ns['Out'])
84 84 self.assertEquals(oldlen, newlen)
85 85 #also test the default caching behavior
86 86 ip.run_cell('1', store_history=True)
87 87 newlen = len(ip.user_ns['Out'])
88 88 self.assertEquals(oldlen+1, newlen)
89 89
90 90 def test_In_variable(self):
91 91 "Verify that In variable grows with user input (GH-284)"
92 92 ip = get_ipython()
93 93 oldlen = len(ip.user_ns['In'])
94 94 ip.run_cell('1;', store_history=True)
95 95 newlen = len(ip.user_ns['In'])
96 96 self.assertEquals(oldlen+1, newlen)
97 97 self.assertEquals(ip.user_ns['In'][-1],'1;')
98 98
99 99 def test_magic_names_in_string(self):
100 100 ip = get_ipython()
101 101 ip.run_cell('a = """\n%exit\n"""')
102 102 self.assertEquals(ip.user_ns['a'], '\n%exit\n')
103 103
104 104 def test_alias_crash(self):
105 105 """Errors in prefilter can't crash IPython"""
106 106 ip = get_ipython()
107 107 ip.run_cell('%alias parts echo first %s second %s')
108 108 # capture stderr:
109 109 save_err = io.stderr
110 110 io.stderr = StringIO()
111 111 ip.run_cell('parts 1')
112 112 err = io.stderr.getvalue()
113 113 io.stderr = save_err
114 114 self.assertEquals(err.split(':')[0], 'ERROR')
115 115
116 116 def test_trailing_newline(self):
117 117 """test that running !(command) does not raise a SyntaxError"""
118 118 ip = get_ipython()
119 119 ip.run_cell('!(true)\n', False)
120 120 ip.run_cell('!(true)\n\n\n', False)
121 121
122 122 def test_gh_597(self):
123 123 """Pretty-printing lists of objects with non-ascii reprs may cause
124 124 problems."""
125 125 class Spam(object):
126 126 def __repr__(self):
127 127 return "\xe9"*50
128 128 import IPython.core.formatters
129 129 f = IPython.core.formatters.PlainTextFormatter()
130 130 f([Spam(),Spam()])
131 131
132 132
133 133 def test_future_flags(self):
134 134 """Check that future flags are used for parsing code (gh-777)"""
135 135 ip = get_ipython()
136 136 ip.run_cell('from __future__ import print_function')
137 137 try:
138 138 ip.run_cell('prfunc_return_val = print(1,2, sep=" ")')
139 139 assert 'prfunc_return_val' in ip.user_ns
140 140 finally:
141 141 # Reset compiler flags so we don't mess up other tests.
142 142 ip.compile.reset_compiler_flags()
143 143
144 144 def test_future_unicode(self):
145 145 """Check that unicode_literals is imported from __future__ (gh #786)"""
146 146 ip = get_ipython()
147 147 try:
148 148 ip.run_cell(u'byte_str = "a"')
149 149 assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default
150 150 ip.run_cell('from __future__ import unicode_literals')
151 151 ip.run_cell(u'unicode_str = "a"')
152 152 assert isinstance(ip.user_ns['unicode_str'], unicode) # strings literals are now unicode
153 153 finally:
154 154 # Reset compiler flags so we don't mess up other tests.
155 155 ip.compile.reset_compiler_flags()
156 156
157 157 def test_can_pickle(self):
158 158 "Can we pickle objects defined interactively (GH-29)"
159 159 ip = get_ipython()
160 160 ip.reset()
161 161 ip.run_cell(("class Mylist(list):\n"
162 162 " def __init__(self,x=[]):\n"
163 163 " list.__init__(self,x)"))
164 164 ip.run_cell("w=Mylist([1,2,3])")
165 165
166 166 from cPickle import dumps
167 167
168 168 # We need to swap in our main module - this is only necessary
169 169 # inside the test framework, because IPython puts the interactive module
170 170 # in place (but the test framework undoes this).
171 171 _main = sys.modules['__main__']
172 172 sys.modules['__main__'] = ip.user_module
173 173 try:
174 174 res = dumps(ip.user_ns["w"])
175 175 finally:
176 176 sys.modules['__main__'] = _main
177 177 self.assertTrue(isinstance(res, bytes))
178 178
179 179 def test_global_ns(self):
180 180 "Code in functions must be able to access variables outside them."
181 181 ip = get_ipython()
182 182 ip.run_cell("a = 10")
183 183 ip.run_cell(("def f(x):\n"
184 184 " return x + a"))
185 185 ip.run_cell("b = f(12)")
186 186 self.assertEqual(ip.user_ns["b"], 22)
187 187
188 188 def test_bad_custom_tb(self):
189 189 """Check that InteractiveShell is protected from bad custom exception handlers"""
190 190 ip = get_ipython()
191 191 from IPython.utils import io
192 192 save_stderr = io.stderr
193 193 try:
194 194 # capture stderr
195 195 io.stderr = StringIO()
196 196 ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
197 197 self.assertEquals(ip.custom_exceptions, (IOError,))
198 198 ip.run_cell(u'raise IOError("foo")')
199 199 self.assertEquals(ip.custom_exceptions, ())
200 200 self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
201 201 finally:
202 202 io.stderr = save_stderr
203 203
204 204 def test_bad_custom_tb_return(self):
205 205 """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
206 206 ip = get_ipython()
207 207 from IPython.utils import io
208 208 save_stderr = io.stderr
209 209 try:
210 210 # capture stderr
211 211 io.stderr = StringIO()
212 212 ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
213 213 self.assertEquals(ip.custom_exceptions, (NameError,))
214 214 ip.run_cell(u'a=abracadabra')
215 215 self.assertEquals(ip.custom_exceptions, ())
216 216 self.assertTrue("Custom TB Handler failed" in io.stderr.getvalue())
217 217 finally:
218 218 io.stderr = save_stderr
219 219
220 220 def test_drop_by_id(self):
221 221 ip = get_ipython()
222 222 myvars = {"a":object(), "b":object(), "c": object()}
223 223 ip.push(myvars, interactive=False)
224 224 for name in myvars:
225 225 assert name in ip.user_ns, name
226 226 assert name in ip.user_ns_hidden, name
227 227 ip.user_ns['b'] = 12
228 228 ip.drop_by_id(myvars)
229 229 for name in ["a", "c"]:
230 230 assert name not in ip.user_ns, name
231 231 assert name not in ip.user_ns_hidden, name
232 232 assert ip.user_ns['b'] == 12
233 233 ip.reset()
234 234
235 235 def test_var_expand(self):
236 236 ip = get_ipython()
237 237 ip.user_ns['f'] = u'Ca\xf1o'
238 238 self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
239 239 self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
240 240 self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
241 241 self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
242 242
243 243 ip.user_ns['f'] = b'Ca\xc3\xb1o'
244 244 # This should not raise any exception:
245 245 ip.var_expand(u'echo $f')
246 246
247 247 def test_bad_var_expand(self):
248 248 """var_expand on invalid formats shouldn't raise"""
249 249 ip = get_ipython()
250 250
251 251 # SyntaxError
252 252 self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
253 253 # NameError
254 254 self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
255 255 # ZeroDivisionError
256 256 self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
257 257
258 258 def test_silent_nopostexec(self):
259 259 """run_cell(silent=True) doesn't invoke post-exec funcs"""
260 260 ip = get_ipython()
261 261
262 262 d = dict(called=False)
263 263 def set_called():
264 264 d['called'] = True
265 265
266 266 ip.register_post_execute(set_called)
267 267 ip.run_cell("1", silent=True)
268 268 self.assertFalse(d['called'])
269 269 # double-check that non-silent exec did what we expected
270 270 # silent to avoid
271 271 ip.run_cell("1")
272 272 self.assertTrue(d['called'])
273 273 # remove post-exec
274 274 ip._post_execute.pop(set_called)
275 275
276 276 def test_silent_noadvance(self):
277 277 """run_cell(silent=True) doesn't advance execution_count"""
278 278 ip = get_ipython()
279 279
280 280 ec = ip.execution_count
281 281 # silent should force store_history=False
282 282 ip.run_cell("1", store_history=True, silent=True)
283 283
284 284 self.assertEquals(ec, ip.execution_count)
285 285 # double-check that non-silent exec did what we expected
286 286 # silent to avoid
287 287 ip.run_cell("1", store_history=True)
288 288 self.assertEquals(ec+1, ip.execution_count)
289 289
290 290 def test_silent_nodisplayhook(self):
291 291 """run_cell(silent=True) doesn't trigger displayhook"""
292 292 ip = get_ipython()
293 293
294 294 d = dict(called=False)
295 295
296 296 trap = ip.display_trap
297 297 save_hook = trap.hook
298 298
299 299 def failing_hook(*args, **kwargs):
300 300 d['called'] = True
301 301
302 302 try:
303 303 trap.hook = failing_hook
304 304 ip.run_cell("1", silent=True)
305 305 self.assertFalse(d['called'])
306 306 # double-check that non-silent exec did what we expected
307 307 # silent to avoid
308 308 ip.run_cell("1")
309 309 self.assertTrue(d['called'])
310 310 finally:
311 311 trap.hook = save_hook
312 312
313 313 @skipif(sys.version_info[0] >= 3, "softspace removed in py3")
314 314 def test_print_softspace(self):
315 315 """Verify that softspace is handled correctly when executing multiple
316 316 statements.
317 317
318 318 In [1]: print 1; print 2
319 319 1
320 320 2
321 321
322 322 In [2]: print 1,; print 2
323 323 1 2
324 324 """
325 325
326 326 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
327 327
328 328 def setUp(self):
329 329 self.BASETESTDIR = tempfile.mkdtemp()
330 330 self.TESTDIR = join(self.BASETESTDIR, u"åäö")
331 331 os.mkdir(self.TESTDIR)
332 332 with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile:
333 333 sfile.write("pass\n")
334 334 self.oldpath = os.getcwdu()
335 335 os.chdir(self.TESTDIR)
336 336 self.fname = u"åäötestscript.py"
337 337
338 338
339 339 def tearDown(self):
340 340 os.chdir(self.oldpath)
341 341 shutil.rmtree(self.BASETESTDIR)
342 342
343 343 def test_1(self):
344 344 """Test safe_execfile with non-ascii path
345 345 """
346 _ip.shell.safe_execfile(self.fname, {}, raise_exceptions=True)
346 _ip.safe_execfile(self.fname, {}, raise_exceptions=True)
347 347
348 348
349 349 class TestSystemRaw(unittest.TestCase):
350 350 def test_1(self):
351 351 """Test system_raw with non-ascii cmd
352 352 """
353 353 cmd = ur'''python -c "'åäö'" '''
354 _ip.shell.system_raw(cmd)
354 _ip.system_raw(cmd)
355 355
356 356
357 357 def test__IPYTHON__():
358 358 # This shouldn't raise a NameError, that's all
359 359 __IPYTHON__
@@ -1,480 +1,480 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for various magic functions.
3 3
4 4 Needs to be run by nose (to make ipython session available).
5 5 """
6 6 from __future__ import absolute_import
7 7
8 8 #-----------------------------------------------------------------------------
9 9 # Imports
10 10 #-----------------------------------------------------------------------------
11 11
12 12 import io
13 13 import os
14 14 import sys
15 15 from StringIO import StringIO
16 16
17 17 import nose.tools as nt
18 18
19 19 from IPython.nbformat.v3.tests.nbexamples import nb0
20 20 from IPython.nbformat import current
21 21 from IPython.testing import decorators as dec
22 22 from IPython.testing import tools as tt
23 23 from IPython.utils import py3compat
24 24 from IPython.utils.tempdir import TemporaryDirectory
25 25
26 26 #-----------------------------------------------------------------------------
27 27 # Test functions begin
28 28 #-----------------------------------------------------------------------------
29 29
30 30
31 31 def test_rehashx():
32 32 # clear up everything
33 33 _ip = get_ipython()
34 34 _ip.alias_manager.alias_table.clear()
35 35 del _ip.db['syscmdlist']
36 36
37 37 _ip.magic('rehashx')
38 38 # Practically ALL ipython development systems will have more than 10 aliases
39 39
40 40 yield (nt.assert_true, len(_ip.alias_manager.alias_table) > 10)
41 41 for key, val in _ip.alias_manager.alias_table.iteritems():
42 42 # we must strip dots from alias names
43 43 nt.assert_true('.' not in key)
44 44
45 45 # rehashx must fill up syscmdlist
46 46 scoms = _ip.db['syscmdlist']
47 47 yield (nt.assert_true, len(scoms) > 10)
48 48
49 49
50 50 def test_magic_parse_options():
51 51 """Test that we don't mangle paths when parsing magic options."""
52 52 ip = get_ipython()
53 53 path = 'c:\\x'
54 opts = ip.parse_options('-f %s' % path,'f:')[0]
54 opts = ip._magic.parse_options('-f %s' % path,'f:')[0]
55 55 # argv splitting is os-dependent
56 56 if os.name == 'posix':
57 57 expected = 'c:x'
58 58 else:
59 59 expected = path
60 60 nt.assert_equals(opts['f'], expected)
61 61
62 62
63 63 @dec.skip_without('sqlite3')
64 64 def doctest_hist_f():
65 65 """Test %hist -f with temporary filename.
66 66
67 67 In [9]: import tempfile
68 68
69 69 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
70 70
71 71 In [11]: %hist -nl -f $tfile 3
72 72
73 73 In [13]: import os; os.unlink(tfile)
74 74 """
75 75
76 76
77 77 @dec.skip_without('sqlite3')
78 78 def doctest_hist_r():
79 79 """Test %hist -r
80 80
81 81 XXX - This test is not recording the output correctly. For some reason, in
82 82 testing mode the raw history isn't getting populated. No idea why.
83 83 Disabling the output checking for now, though at least we do run it.
84 84
85 85 In [1]: 'hist' in _ip.lsmagic()
86 86 Out[1]: True
87 87
88 88 In [2]: x=1
89 89
90 90 In [3]: %hist -rl 2
91 91 x=1 # random
92 92 %hist -r 2
93 93 """
94 94
95 95
96 96 @dec.skip_without('sqlite3')
97 97 def doctest_hist_op():
98 98 """Test %hist -op
99 99
100 100 In [1]: class b(float):
101 101 ...: pass
102 102 ...:
103 103
104 104 In [2]: class s(object):
105 105 ...: def __str__(self):
106 106 ...: return 's'
107 107 ...:
108 108
109 109 In [3]:
110 110
111 111 In [4]: class r(b):
112 112 ...: def __repr__(self):
113 113 ...: return 'r'
114 114 ...:
115 115
116 116 In [5]: class sr(s,r): pass
117 117 ...:
118 118
119 119 In [6]:
120 120
121 121 In [7]: bb=b()
122 122
123 123 In [8]: ss=s()
124 124
125 125 In [9]: rr=r()
126 126
127 127 In [10]: ssrr=sr()
128 128
129 129 In [11]: 4.5
130 130 Out[11]: 4.5
131 131
132 132 In [12]: str(ss)
133 133 Out[12]: 's'
134 134
135 135 In [13]:
136 136
137 137 In [14]: %hist -op
138 138 >>> class b:
139 139 ... pass
140 140 ...
141 141 >>> class s(b):
142 142 ... def __str__(self):
143 143 ... return 's'
144 144 ...
145 145 >>>
146 146 >>> class r(b):
147 147 ... def __repr__(self):
148 148 ... return 'r'
149 149 ...
150 150 >>> class sr(s,r): pass
151 151 >>>
152 152 >>> bb=b()
153 153 >>> ss=s()
154 154 >>> rr=r()
155 155 >>> ssrr=sr()
156 156 >>> 4.5
157 157 4.5
158 158 >>> str(ss)
159 159 's'
160 160 >>>
161 161 """
162 162
163 163
164 164 @dec.skip_without('sqlite3')
165 165 def test_macro():
166 166 ip = get_ipython()
167 167 ip.history_manager.reset() # Clear any existing history.
168 168 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
169 169 for i, cmd in enumerate(cmds, start=1):
170 170 ip.history_manager.store_inputs(i, cmd)
171 171 ip.magic("macro test 1-3")
172 172 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
173 173
174 174 # List macros.
175 175 assert "test" in ip.magic("macro")
176 176
177 177
178 178 @dec.skip_without('sqlite3')
179 179 def test_macro_run():
180 180 """Test that we can run a multi-line macro successfully."""
181 181 ip = get_ipython()
182 182 ip.history_manager.reset()
183 183 cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
184 184 "%macro test 2-3"]
185 185 for cmd in cmds:
186 186 ip.run_cell(cmd, store_history=True)
187 187 nt.assert_equal(ip.user_ns["test"].value,
188 188 py3compat.doctest_refactor_print("a+=1\nprint a\n"))
189 189 with tt.AssertPrints("12"):
190 190 ip.run_cell("test")
191 191 with tt.AssertPrints("13"):
192 192 ip.run_cell("test")
193 193
194 194
195 195 @dec.skipif_not_numpy
196 196 def test_numpy_reset_array_undec():
197 197 "Test '%reset array' functionality"
198 198 _ip.ex('import numpy as np')
199 199 _ip.ex('a = np.empty(2)')
200 200 yield (nt.assert_true, 'a' in _ip.user_ns)
201 201 _ip.magic('reset -f array')
202 202 yield (nt.assert_false, 'a' in _ip.user_ns)
203 203
204 204 def test_reset_out():
205 205 "Test '%reset out' magic"
206 206 _ip.run_cell("parrot = 'dead'", store_history=True)
207 207 # test '%reset -f out', make an Out prompt
208 208 _ip.run_cell("parrot", store_history=True)
209 209 nt.assert_true('dead' in [_ip.user_ns[x] for x in '_','__','___'])
210 210 _ip.magic('reset -f out')
211 211 nt.assert_false('dead' in [_ip.user_ns[x] for x in '_','__','___'])
212 212 nt.assert_true(len(_ip.user_ns['Out']) == 0)
213 213
214 214 def test_reset_in():
215 215 "Test '%reset in' magic"
216 216 # test '%reset -f in'
217 217 _ip.run_cell("parrot", store_history=True)
218 218 nt.assert_true('parrot' in [_ip.user_ns[x] for x in '_i','_ii','_iii'])
219 219 _ip.magic('%reset -f in')
220 220 nt.assert_false('parrot' in [_ip.user_ns[x] for x in '_i','_ii','_iii'])
221 221 nt.assert_true(len(set(_ip.user_ns['In'])) == 1)
222 222
223 223 def test_reset_dhist():
224 224 "Test '%reset dhist' magic"
225 225 _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
226 226 _ip.magic('cd ' + os.path.dirname(nt.__file__))
227 227 _ip.magic('cd -')
228 228 nt.assert_true(len(_ip.user_ns['_dh']) > 0)
229 229 _ip.magic('reset -f dhist')
230 230 nt.assert_true(len(_ip.user_ns['_dh']) == 0)
231 231 _ip.run_cell("_dh = [d for d in tmp]") #restore
232 232
233 233 def test_reset_in_length():
234 234 "Test that '%reset in' preserves In[] length"
235 235 _ip.run_cell("print 'foo'")
236 236 _ip.run_cell("reset -f in")
237 237 nt.assert_true(len(_ip.user_ns['In']) == _ip.displayhook.prompt_count+1)
238 238
239 239 def test_time():
240 240 _ip.magic('time None')
241 241
242 242 def test_tb_syntaxerror():
243 243 """test %tb after a SyntaxError"""
244 244 ip = get_ipython()
245 245 ip.run_cell("for")
246 246
247 247 # trap and validate stdout
248 248 save_stdout = sys.stdout
249 249 try:
250 250 sys.stdout = StringIO()
251 251 ip.run_cell("%tb")
252 252 out = sys.stdout.getvalue()
253 253 finally:
254 254 sys.stdout = save_stdout
255 255 # trim output, and only check the last line
256 256 last_line = out.rstrip().splitlines()[-1].strip()
257 257 nt.assert_equals(last_line, "SyntaxError: invalid syntax")
258 258
259 259
260 260 @py3compat.doctest_refactor_print
261 261 def doctest_time():
262 262 """
263 263 In [10]: %time None
264 264 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
265 265 Wall time: 0.00 s
266 266
267 267 In [11]: def f(kmjy):
268 268 ....: %time print 2*kmjy
269 269
270 270 In [12]: f(3)
271 271 6
272 272 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
273 273 Wall time: 0.00 s
274 274 """
275 275
276 276
277 277 def test_doctest_mode():
278 278 "Toggle doctest_mode twice, it should be a no-op and run without error"
279 279 _ip.magic('doctest_mode')
280 280 _ip.magic('doctest_mode')
281 281
282 282
283 283 def test_parse_options():
284 284 """Tests for basic options parsing in magics."""
285 285 # These are only the most minimal of tests, more should be added later. At
286 286 # the very least we check that basic text/unicode calls work OK.
287 nt.assert_equal(_ip.parse_options('foo', '')[1], 'foo')
288 nt.assert_equal(_ip.parse_options(u'foo', '')[1], u'foo')
287 nt.assert_equal(_ip._magic.parse_options('foo', '')[1], 'foo')
288 nt.assert_equal(_ip._magic.parse_options(u'foo', '')[1], u'foo')
289 289
290 290
291 291 def test_dirops():
292 292 """Test various directory handling operations."""
293 293 # curpath = lambda :os.path.splitdrive(os.getcwdu())[1].replace('\\','/')
294 294 curpath = os.getcwdu
295 295 startdir = os.getcwdu()
296 296 ipdir = os.path.realpath(_ip.ipython_dir)
297 297 try:
298 298 _ip.magic('cd "%s"' % ipdir)
299 299 nt.assert_equal(curpath(), ipdir)
300 300 _ip.magic('cd -')
301 301 nt.assert_equal(curpath(), startdir)
302 302 _ip.magic('pushd "%s"' % ipdir)
303 303 nt.assert_equal(curpath(), ipdir)
304 304 _ip.magic('popd')
305 305 nt.assert_equal(curpath(), startdir)
306 306 finally:
307 307 os.chdir(startdir)
308 308
309 309
310 310 def test_xmode():
311 311 # Calling xmode three times should be a no-op
312 312 xmode = _ip.InteractiveTB.mode
313 313 for i in range(3):
314 314 _ip.magic("xmode")
315 315 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
316 316
317 317 def test_reset_hard():
318 318 monitor = []
319 319 class A(object):
320 320 def __del__(self):
321 321 monitor.append(1)
322 322 def __repr__(self):
323 323 return "<A instance>"
324 324
325 325 _ip.user_ns["a"] = A()
326 326 _ip.run_cell("a")
327 327
328 328 nt.assert_equal(monitor, [])
329 _ip.magic_reset("-f")
329 _ip.magic("reset -f")
330 330 nt.assert_equal(monitor, [1])
331 331
332 332 class TestXdel(tt.TempFileMixin):
333 333 def test_xdel(self):
334 334 """Test that references from %run are cleared by xdel."""
335 335 src = ("class A(object):\n"
336 336 " monitor = []\n"
337 337 " def __del__(self):\n"
338 338 " self.monitor.append(1)\n"
339 339 "a = A()\n")
340 340 self.mktmp(src)
341 341 # %run creates some hidden references...
342 342 _ip.magic("run %s" % self.fname)
343 343 # ... as does the displayhook.
344 344 _ip.run_cell("a")
345 345
346 346 monitor = _ip.user_ns["A"].monitor
347 347 nt.assert_equal(monitor, [])
348 348
349 349 _ip.magic("xdel a")
350 350
351 351 # Check that a's __del__ method has been called.
352 352 nt.assert_equal(monitor, [1])
353 353
354 354 def doctest_who():
355 355 """doctest for %who
356 356
357 357 In [1]: %reset -f
358 358
359 359 In [2]: alpha = 123
360 360
361 361 In [3]: beta = 'beta'
362 362
363 363 In [4]: %who int
364 364 alpha
365 365
366 366 In [5]: %who str
367 367 beta
368 368
369 369 In [6]: %whos
370 370 Variable Type Data/Info
371 371 ----------------------------
372 372 alpha int 123
373 373 beta str beta
374 374
375 375 In [7]: %who_ls
376 376 Out[7]: ['alpha', 'beta']
377 377 """
378 378
379 379 def test_whos():
380 380 """Check that whos is protected against objects where repr() fails."""
381 381 class A(object):
382 382 def __repr__(self):
383 383 raise Exception()
384 384 _ip.user_ns['a'] = A()
385 385 _ip.magic("whos")
386 386
387 387 @py3compat.u_format
388 388 def doctest_precision():
389 389 """doctest for %precision
390 390
391 In [1]: f = get_ipython().shell.display_formatter.formatters['text/plain']
391 In [1]: f = get_ipython().display_formatter.formatters['text/plain']
392 392
393 393 In [2]: %precision 5
394 394 Out[2]: {u}'%.5f'
395 395
396 396 In [3]: f.float_format
397 397 Out[3]: {u}'%.5f'
398 398
399 399 In [4]: %precision %e
400 400 Out[4]: {u}'%e'
401 401
402 402 In [5]: f(3.1415927)
403 403 Out[5]: {u}'3.141593e+00'
404 404 """
405 405
406 406 def test_psearch():
407 407 with tt.AssertPrints("dict.fromkeys"):
408 408 _ip.run_cell("dict.fr*?")
409 409
410 410 def test_timeit_shlex():
411 411 """test shlex issues with timeit (#1109)"""
412 412 _ip.ex("def f(*a,**kw): pass")
413 413 _ip.magic('timeit -n1 "this is a bug".count(" ")')
414 414 _ip.magic('timeit -r1 -n1 f(" ", 1)')
415 415 _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
416 416 _ip.magic('timeit -r1 -n1 ("a " + "b")')
417 417 _ip.magic('timeit -r1 -n1 f("a " + "b")')
418 418 _ip.magic('timeit -r1 -n1 f("a " + "b ")')
419 419
420 420
421 421 def test_timeit_arguments():
422 422 "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
423 423 _ip.magic("timeit ('#')")
424 424
425 @dec.skipif(_ip.magic_prun == _ip.profile_missing_notice)
425 @dec.skipif(_ip._magic.magic_prun == _ip._magic.profile_missing_notice)
426 426 def test_prun_quotes():
427 427 "Test that prun does not clobber string escapes (GH #1302)"
428 428 _ip.magic("prun -q x = '\t'")
429 429 nt.assert_equal(_ip.user_ns['x'], '\t')
430 430
431 431 def test_extension():
432 432 tmpdir = TemporaryDirectory()
433 433 orig_ipython_dir = _ip.ipython_dir
434 434 try:
435 435 _ip.ipython_dir = tmpdir.name
436 436 nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
437 437 url = os.path.join(os.path.dirname(__file__), "daft_extension.py")
438 438 _ip.magic("install_ext %s" % url)
439 439 _ip.user_ns.pop('arq', None)
440 440 _ip.magic("load_ext daft_extension")
441 441 tt.assert_equal(_ip.user_ns['arq'], 185)
442 442 _ip.magic("unload_ext daft_extension")
443 443 assert 'arq' not in _ip.user_ns
444 444 finally:
445 445 _ip.ipython_dir = orig_ipython_dir
446 446
447 447 def test_notebook_export_json():
448 448 with TemporaryDirectory() as td:
449 449 outfile = os.path.join(td, "nb.ipynb")
450 450 _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
451 451 _ip.magic("notebook -e %s" % outfile)
452 452
453 453 def test_notebook_export_py():
454 454 with TemporaryDirectory() as td:
455 455 outfile = os.path.join(td, "nb.py")
456 456 _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
457 457 _ip.magic("notebook -e %s" % outfile)
458 458
459 459 def test_notebook_reformat_py():
460 460 with TemporaryDirectory() as td:
461 461 infile = os.path.join(td, "nb.ipynb")
462 462 with io.open(infile, 'w', encoding='utf-8') as f:
463 463 current.write(nb0, f, 'json')
464 464
465 465 _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
466 466 _ip.magic("notebook -f py %s" % infile)
467 467
468 468 def test_notebook_reformat_json():
469 469 with TemporaryDirectory() as td:
470 470 infile = os.path.join(td, "nb.py")
471 471 with io.open(infile, 'w', encoding='utf-8') as f:
472 472 current.write(nb0, f, 'py')
473 473
474 474 _ip.ex(py3compat.u_format(u"u = {u}'héllo'"))
475 475 _ip.magic("notebook -f ipynb %s" % infile)
476 476 _ip.magic("notebook -f json %s" % infile)
477 477
478 478 def test_env():
479 479 env = _ip.magic("env")
480 480 assert isinstance(env, dict), type(env)
@@ -1,110 +1,110 b''
1 1 """Tests for input manipulation machinery."""
2 2
3 3 #-----------------------------------------------------------------------------
4 4 # Imports
5 5 #-----------------------------------------------------------------------------
6 6 import nose.tools as nt
7 7
8 8 from IPython.core.prefilter import AutocallChecker
9 9 from IPython.testing import tools as tt, decorators as dec
10 10 from IPython.testing.globalipapp import get_ipython
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Tests
14 14 #-----------------------------------------------------------------------------
15 15 ip = get_ipython()
16 16
17 17 @dec.parametric
18 18 def test_prefilter():
19 19 """Test user input conversions"""
20 20
21 21 # pairs of (raw, expected correct) input
22 22 pairs = [ ('2+2','2+2'),
23 23 ('>>> 2+2','2+2'),
24 24 ('>>> # This is a comment\n'
25 25 '... 2+2',
26 26 '# This is a comment\n'
27 27 '2+2'),
28 28 # Some IPython input
29 29 ('In [1]: 1', '1'),
30 30 ('In [2]: for i in range(5):\n'
31 31 ' ...: print i,',
32 32 'for i in range(5):\n'
33 33 ' print i,'),
34 34 ]
35 35
36 36 for raw, correct in pairs:
37 37 yield nt.assert_equals(ip.prefilter(raw), correct)
38 38
39 39
40 40 @dec.parametric
41 41 def test_autocall_binops():
42 42 """See https://github.com/ipython/ipython/issues/81"""
43 43 ip.magic('autocall 2')
44 44 f = lambda x: x
45 45 ip.user_ns['f'] = f
46 46 try:
47 47 yield nt.assert_equals(ip.prefilter('f 1'),'f(1)')
48 48 for t in ['f +1', 'f -1']:
49 49 yield nt.assert_equals(ip.prefilter(t), t)
50 50
51 51 # Run tests again with a more permissive exclude_regexp, which will
52 52 # allow transformation of binary operations ('f -1' -> 'f(-1)').
53 53 pm = ip.prefilter_manager
54 54 ac = AutocallChecker(shell=pm.shell, prefilter_manager=pm,
55 55 config=pm.config)
56 56 try:
57 57 ac.priority = 1
58 58 ac.exclude_regexp = r'^[,&^\|\*/]|^is |^not |^in |^and |^or '
59 59 pm.sort_checkers()
60 60
61 61 yield nt.assert_equals(ip.prefilter('f -1'), 'f(-1)')
62 62 yield nt.assert_equals(ip.prefilter('f +1'), 'f(+1)')
63 63 finally:
64 64 pm.unregister_checker(ac)
65 65 finally:
66 66 ip.magic('autocall 0')
67 67 del ip.user_ns['f']
68 68
69 69
70 70 @dec.parametric
71 71 def test_issue_114():
72 72 """Check that multiline string literals don't expand as magic
73 73 see http://github.com/ipython/ipython/issues/114"""
74 74
75 75 template = '"""\n%s\n"""'
76 76 # Store the current value of multi_line_specials and turn it off before
77 77 # running test, since it could be true (case in which the test doesn't make
78 78 # sense, as multiline string literals *will* expand as magic in that case).
79 79 msp = ip.prefilter_manager.multi_line_specials
80 80 ip.prefilter_manager.multi_line_specials = False
81 81 try:
82 for mgk in ip.lsmagic():
82 for mgk in ip._magic.lsmagic():
83 83 raw = template % mgk
84 84 yield nt.assert_equals(ip.prefilter(raw), raw)
85 85 finally:
86 86 ip.prefilter_manager.multi_line_specials = msp
87 87
88 88
89 89 def test_prefilter_attribute_errors():
90 90 """Capture exceptions thrown by user objects on attribute access.
91 91
92 92 See http://github.com/ipython/ipython/issues/988."""
93 93
94 94 class X(object):
95 95 def __getattr__(self, k):
96 96 raise ValueError('broken object')
97 97 def __call__(self, x):
98 98 return x
99 99
100 100 # Create a callable broken object
101 101 ip.user_ns['x'] = X()
102 102 ip.magic('autocall 2')
103 103 try:
104 104 # Even if x throws an attribute error when looking at its rewrite
105 105 # attribute, we should not crash. So the test here is simply making
106 106 # the prefilter call and not having an exception.
107 107 ip.prefilter('x 1')
108 108 finally:
109 109 del ip.user_ns['x']
110 110 ip.magic('autocall 0')
@@ -1,669 +1,683 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Subclass of InteractiveShell for terminal based frontends."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 #-----------------------------------------------------------------------------
14 14 # Imports
15 15 #-----------------------------------------------------------------------------
16 16
17 17 import __builtin__
18 18 import bdb
19 19 import os
20 20 import re
21 21 import sys
22 22 import textwrap
23 23
24 24 try:
25 25 from contextlib import nested
26 26 except:
27 27 from IPython.utils.nested_context import nested
28 28
29 29 from IPython.core.error import TryNext, UsageError
30 30 from IPython.core.usage import interactive_usage, default_banner
31 31 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
32 32 from IPython.core.pylabtools import pylab_activate
33 33 from IPython.testing.skipdoctest import skip_doctest
34 34 from IPython.utils.encoding import get_stream_enc
35 35 from IPython.utils import py3compat
36 36 from IPython.utils.terminal import toggle_set_term_title, set_term_title
37 37 from IPython.utils.process import abbrev_cwd
38 38 from IPython.utils.warn import warn, error
39 39 from IPython.utils.text import num_ini_spaces, SList
40 40 from IPython.utils.traitlets import Integer, CBool, Unicode
41 41
42 42 #-----------------------------------------------------------------------------
43 43 # Utilities
44 44 #-----------------------------------------------------------------------------
45 45
46 46 def get_default_editor():
47 47 try:
48 48 ed = os.environ['EDITOR']
49 49 except KeyError:
50 50 if os.name == 'posix':
51 51 ed = 'vi' # the only one guaranteed to be there!
52 52 else:
53 53 ed = 'notepad' # same in Windows!
54 54 return ed
55 55
56 56
57 57 def get_pasted_lines(sentinel, l_input=py3compat.input):
58 58 """ Yield pasted lines until the user enters the given sentinel value.
59 59 """
60 60 print "Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
61 61 % sentinel
62 62 while True:
63 63 try:
64 64 l = l_input(':')
65 65 if l == sentinel:
66 66 return
67 67 else:
68 68 yield l
69 69 except EOFError:
70 70 print '<EOF>'
71 71 return
72 72
73 73
74 74 def strip_email_quotes(raw_lines):
75 75 """ Strip email quotation marks at the beginning of each line.
76 76
77 77 We don't do any more input transofrmations here because the main shell's
78 78 prefiltering handles other cases.
79 79 """
80 80 lines = [re.sub(r'^\s*(\s?>)+', '', l) for l in raw_lines]
81 81 return '\n'.join(lines) + '\n'
82 82
83 83
84 84 # These two functions are needed by the %paste/%cpaste magics. In practice
85 85 # they are basically methods (they take the shell as their first argument), but
86 86 # we leave them as standalone functions because eventually the magics
87 87 # themselves will become separate objects altogether. At that point, the
88 88 # magics will have access to the shell object, and these functions can be made
89 89 # methods of the magic object, but not of the shell.
90 90
91 91 def store_or_execute(shell, block, name):
92 92 """ Execute a block, or store it in a variable, per the user's request.
93 93 """
94 94 # Dedent and prefilter so what we store matches what is executed by
95 95 # run_cell.
96 96 b = shell.prefilter(textwrap.dedent(block))
97 97
98 98 if name:
99 99 # If storing it for further editing, run the prefilter on it
100 100 shell.user_ns[name] = SList(b.splitlines())
101 101 print "Block assigned to '%s'" % name
102 102 else:
103 103 shell.user_ns['pasted_block'] = b
104 104 shell.run_cell(b)
105 105
106 106
107 107 def rerun_pasted(shell, name='pasted_block'):
108 108 """ Rerun a previously pasted command.
109 109 """
110 110 b = shell.user_ns.get(name)
111 111
112 112 # Sanity checks
113 113 if b is None:
114 114 raise UsageError('No previous pasted block available')
115 115 if not isinstance(b, basestring):
116 116 raise UsageError(
117 117 "Variable 'pasted_block' is not a string, can't execute")
118 118
119 119 print "Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b))
120 120 shell.run_cell(b)
121 121
122 122
123 #------------------------------------------------------------------------
124 # Terminal-specific magics
125 #------------------------------------------------------------------------
126
127 def magic_autoindent(self, parameter_s = ''):
128 """Toggle autoindent on/off (if available)."""
129
130 self.shell.set_autoindent()
131 print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
132
133 @skip_doctest
134 def magic_cpaste(self, parameter_s=''):
135 """Paste & execute a pre-formatted code block from clipboard.
136
137 You must terminate the block with '--' (two minus-signs) or Ctrl-D
138 alone on the line. You can also provide your own sentinel with '%paste
139 -s %%' ('%%' is the new sentinel for this operation)
140
141 The block is dedented prior to execution to enable execution of method
142 definitions. '>' and '+' characters at the beginning of a line are
143 ignored, to allow pasting directly from e-mails, diff files and
144 doctests (the '...' continuation prompt is also stripped). The
145 executed block is also assigned to variable named 'pasted_block' for
146 later editing with '%edit pasted_block'.
147
148 You can also pass a variable name as an argument, e.g. '%cpaste foo'.
149 This assigns the pasted block to variable 'foo' as string, without
150 dedenting or executing it (preceding >>> and + is still stripped)
151
152 '%cpaste -r' re-executes the block previously entered by cpaste.
153
154 Do not be alarmed by garbled output on Windows (it's a readline bug).
155 Just press enter and type -- (and press enter again) and the block
156 will be what was just pasted.
157
158 IPython statements (magics, shell escapes) are not supported (yet).
159
160 See also
161 --------
162 paste: automatically pull code from clipboard.
163
164 Examples
165 --------
166 ::
167
168 In [8]: %cpaste
169 Pasting code; enter '--' alone on the line to stop.
170 :>>> a = ["world!", "Hello"]
171 :>>> print " ".join(sorted(a))
172 :--
173 Hello world!
174 """
175
176 opts, name = self.parse_options(parameter_s, 'rs:', mode='string')
177 if 'r' in opts:
178 rerun_pasted(self.shell)
179 return
180
181 sentinel = opts.get('s', '--')
182 block = strip_email_quotes(get_pasted_lines(sentinel))
183 store_or_execute(self.shell, block, name)
184
185
186 def magic_paste(self, parameter_s=''):
187 """Paste & execute a pre-formatted code block from clipboard.
188
189 The text is pulled directly from the clipboard without user
190 intervention and printed back on the screen before execution (unless
191 the -q flag is given to force quiet mode).
192
193 The block is dedented prior to execution to enable execution of method
194 definitions. '>' and '+' characters at the beginning of a line are
195 ignored, to allow pasting directly from e-mails, diff files and
196 doctests (the '...' continuation prompt is also stripped). The
197 executed block is also assigned to variable named 'pasted_block' for
198 later editing with '%edit pasted_block'.
199
200 You can also pass a variable name as an argument, e.g. '%paste foo'.
201 This assigns the pasted block to variable 'foo' as string, without
202 dedenting or executing it (preceding >>> and + is still stripped)
203
204 Options
205 -------
206
207 -r: re-executes the block previously entered by cpaste.
208
209 -q: quiet mode: do not echo the pasted text back to the terminal.
210
211 IPython statements (magics, shell escapes) are not supported (yet).
212
213 See also
214 --------
215 cpaste: manually paste code into terminal until you mark its end.
216 """
217 opts, name = self.parse_options(parameter_s, 'rq', mode='string')
218 if 'r' in opts:
219 rerun_pasted(self.shell)
220 return
221 try:
222 text = self.shell.hooks.clipboard_get()
223 block = strip_email_quotes(text.splitlines())
224 except TryNext as clipboard_exc:
225 message = getattr(clipboard_exc, 'args')
226 if message:
227 error(message[0])
228 else:
229 error('Could not get text from the clipboard.')
230 return
231
232 # By default, echo back to terminal unless quiet mode is requested
233 if 'q' not in opts:
234 write = self.shell.write
235 write(self.shell.pycolorize(block))
236 if not block.endswith('\n'):
237 write('\n')
238 write("## -- End pasted text --\n")
239
240 store_or_execute(self.shell, block, name)
241
242
243 # Class-level: add a '%cls' magic only on Windows
244 if sys.platform == 'win32':
245 def magic_cls(self, s):
246 """Clear screen.
247 """
248 os.system("cls")
249
123 250 #-----------------------------------------------------------------------------
124 251 # Main class
125 252 #-----------------------------------------------------------------------------
126 253
127 254 class TerminalInteractiveShell(InteractiveShell):
128 255
129 256 autoedit_syntax = CBool(False, config=True,
130 257 help="auto editing of files with syntax errors.")
131 258 banner = Unicode('')
132 259 banner1 = Unicode(default_banner, config=True,
133 260 help="""The part of the banner to be printed before the profile"""
134 261 )
135 262 banner2 = Unicode('', config=True,
136 263 help="""The part of the banner to be printed after the profile"""
137 264 )
138 265 confirm_exit = CBool(True, config=True,
139 266 help="""
140 267 Set to confirm when you try to exit IPython with an EOF (Control-D
141 268 in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
142 269 you can force a direct exit without any confirmation.""",
143 270 )
144 271 # This display_banner only controls whether or not self.show_banner()
145 272 # is called when mainloop/interact are called. The default is False
146 273 # because for the terminal based application, the banner behavior
147 274 # is controlled by Global.display_banner, which IPythonApp looks at
148 275 # to determine if *it* should call show_banner() by hand or not.
149 276 display_banner = CBool(False) # This isn't configurable!
150 277 embedded = CBool(False)
151 278 embedded_active = CBool(False)
152 279 editor = Unicode(get_default_editor(), config=True,
153 280 help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
154 281 )
155 282 pager = Unicode('less', config=True,
156 283 help="The shell program to be used for paging.")
157 284
158 285 screen_length = Integer(0, config=True,
159 286 help=
160 287 """Number of lines of your screen, used to control printing of very
161 288 long strings. Strings longer than this number of lines will be sent
162 289 through a pager instead of directly printed. The default value for
163 290 this is 0, which means IPython will auto-detect your screen size every
164 291 time it needs to print certain potentially long strings (this doesn't
165 292 change the behavior of the 'print' keyword, it's only triggered
166 293 internally). If for some reason this isn't working well (it needs
167 294 curses support), specify it yourself. Otherwise don't change the
168 295 default.""",
169 296 )
170 297 term_title = CBool(False, config=True,
171 298 help="Enable auto setting the terminal title."
172 299 )
173 300
174 301 # In the terminal, GUI control is done via PyOS_InputHook
175 302 from IPython.lib.inputhook import enable_gui
176 303 enable_gui = staticmethod(enable_gui)
177 304
178 305 def __init__(self, config=None, ipython_dir=None, profile_dir=None,
179 306 user_ns=None, user_module=None, custom_exceptions=((),None),
180 307 usage=None, banner1=None, banner2=None, display_banner=None):
181 308
182 309 super(TerminalInteractiveShell, self).__init__(
183 310 config=config, profile_dir=profile_dir, user_ns=user_ns,
184 311 user_module=user_module, custom_exceptions=custom_exceptions
185 312 )
186 313 # use os.system instead of utils.process.system by default,
187 314 # because piped system doesn't make sense in the Terminal:
188 315 self.system = self.system_raw
189 316
190 317 self.init_term_title()
191 318 self.init_usage(usage)
192 319 self.init_banner(banner1, banner2, display_banner)
193 320
194 321 #-------------------------------------------------------------------------
195 322 # Things related to the terminal
196 323 #-------------------------------------------------------------------------
197 324
198 325 @property
199 326 def usable_screen_length(self):
200 327 if self.screen_length == 0:
201 328 return 0
202 329 else:
203 330 num_lines_bot = self.separate_in.count('\n')+1
204 331 return self.screen_length - num_lines_bot
205 332
206 333 def init_term_title(self):
207 334 # Enable or disable the terminal title.
208 335 if self.term_title:
209 336 toggle_set_term_title(True)
210 337 set_term_title('IPython: ' + abbrev_cwd())
211 338 else:
212 339 toggle_set_term_title(False)
213 340
214 341 #-------------------------------------------------------------------------
215 342 # Things related to aliases
216 343 #-------------------------------------------------------------------------
217 344
218 345 def init_alias(self):
219 346 # The parent class defines aliases that can be safely used with any
220 347 # frontend.
221 348 super(TerminalInteractiveShell, self).init_alias()
222 349
223 350 # Now define aliases that only make sense on the terminal, because they
224 351 # need direct access to the console in a way that we can't emulate in
225 352 # GUI or web frontend
226 353 if os.name == 'posix':
227 354 aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
228 355 ('man', 'man')]
229 356 elif os.name == 'nt':
230 357 aliases = [('cls', 'cls')]
231 358
232 359
233 360 for name, cmd in aliases:
234 361 self.alias_manager.define_alias(name, cmd)
235 362
236 363 #-------------------------------------------------------------------------
237 364 # Things related to the banner and usage
238 365 #-------------------------------------------------------------------------
239 366
240 367 def _banner1_changed(self):
241 368 self.compute_banner()
242 369
243 370 def _banner2_changed(self):
244 371 self.compute_banner()
245 372
246 373 def _term_title_changed(self, name, new_value):
247 374 self.init_term_title()
248 375
249 376 def init_banner(self, banner1, banner2, display_banner):
250 377 if banner1 is not None:
251 378 self.banner1 = banner1
252 379 if banner2 is not None:
253 380 self.banner2 = banner2
254 381 if display_banner is not None:
255 382 self.display_banner = display_banner
256 383 self.compute_banner()
257 384
258 385 def show_banner(self, banner=None):
259 386 if banner is None:
260 387 banner = self.banner
261 388 self.write(banner)
262 389
263 390 def compute_banner(self):
264 391 self.banner = self.banner1
265 392 if self.profile and self.profile != 'default':
266 393 self.banner += '\nIPython profile: %s\n' % self.profile
267 394 if self.banner2:
268 395 self.banner += '\n' + self.banner2
269 396
270 397 def init_usage(self, usage=None):
271 398 if usage is None:
272 399 self.usage = interactive_usage
273 400 else:
274 401 self.usage = usage
275 402
276 403 #-------------------------------------------------------------------------
277 404 # Mainloop and code execution logic
278 405 #-------------------------------------------------------------------------
279 406
280 407 def mainloop(self, display_banner=None):
281 408 """Start the mainloop.
282 409
283 410 If an optional banner argument is given, it will override the
284 411 internally created default banner.
285 412 """
286 413
287 414 with nested(self.builtin_trap, self.display_trap):
288 415
289 416 while 1:
290 417 try:
291 418 self.interact(display_banner=display_banner)
292 419 #self.interact_with_readline()
293 420 # XXX for testing of a readline-decoupled repl loop, call
294 421 # interact_with_readline above
295 422 break
296 423 except KeyboardInterrupt:
297 424 # this should not be necessary, but KeyboardInterrupt
298 425 # handling seems rather unpredictable...
299 426 self.write("\nKeyboardInterrupt in interact()\n")
300 427
301 428 def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
302 429 """Store multiple lines as a single entry in history"""
303 430
304 431 # do nothing without readline or disabled multiline
305 432 if not self.has_readline or not self.multiline_history:
306 433 return hlen_before_cell
307 434
308 435 # windows rl has no remove_history_item
309 436 if not hasattr(self.readline, "remove_history_item"):
310 437 return hlen_before_cell
311 438
312 439 # skip empty cells
313 440 if not source_raw.rstrip():
314 441 return hlen_before_cell
315 442
316 443 # nothing changed do nothing, e.g. when rl removes consecutive dups
317 444 hlen = self.readline.get_current_history_length()
318 445 if hlen == hlen_before_cell:
319 446 return hlen_before_cell
320 447
321 448 for i in range(hlen - hlen_before_cell):
322 449 self.readline.remove_history_item(hlen - i - 1)
323 450 stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
324 451 self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),
325 452 stdin_encoding))
326 453 return self.readline.get_current_history_length()
327 454
328 455 def interact(self, display_banner=None):
329 456 """Closely emulate the interactive Python console."""
330 457
331 458 # batch run -> do not interact
332 459 if self.exit_now:
333 460 return
334 461
335 462 if display_banner is None:
336 463 display_banner = self.display_banner
337 464
338 465 if isinstance(display_banner, basestring):
339 466 self.show_banner(display_banner)
340 467 elif display_banner:
341 468 self.show_banner()
342 469
343 470 more = False
344 471
345 472 if self.has_readline:
346 473 self.readline_startup_hook(self.pre_readline)
347 474 hlen_b4_cell = self.readline.get_current_history_length()
348 475 else:
349 476 hlen_b4_cell = 0
350 477 # exit_now is set by a call to %Exit or %Quit, through the
351 478 # ask_exit callback.
352 479
353 480 while not self.exit_now:
354 481 self.hooks.pre_prompt_hook()
355 482 if more:
356 483 try:
357 484 prompt = self.prompt_manager.render('in2')
358 485 except:
359 486 self.showtraceback()
360 487 if self.autoindent:
361 488 self.rl_do_indent = True
362 489
363 490 else:
364 491 try:
365 492 prompt = self.separate_in + self.prompt_manager.render('in')
366 493 except:
367 494 self.showtraceback()
368 495 try:
369 496 line = self.raw_input(prompt)
370 497 if self.exit_now:
371 498 # quick exit on sys.std[in|out] close
372 499 break
373 500 if self.autoindent:
374 501 self.rl_do_indent = False
375 502
376 503 except KeyboardInterrupt:
377 504 #double-guard against keyboardinterrupts during kbdint handling
378 505 try:
379 506 self.write('\nKeyboardInterrupt\n')
380 507 source_raw = self.input_splitter.source_raw_reset()[1]
381 508 hlen_b4_cell = \
382 509 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
383 510 more = False
384 511 except KeyboardInterrupt:
385 512 pass
386 513 except EOFError:
387 514 if self.autoindent:
388 515 self.rl_do_indent = False
389 516 if self.has_readline:
390 517 self.readline_startup_hook(None)
391 518 self.write('\n')
392 519 self.exit()
393 520 except bdb.BdbQuit:
394 521 warn('The Python debugger has exited with a BdbQuit exception.\n'
395 522 'Because of how pdb handles the stack, it is impossible\n'
396 523 'for IPython to properly format this particular exception.\n'
397 524 'IPython will resume normal operation.')
398 525 except:
399 526 # exceptions here are VERY RARE, but they can be triggered
400 527 # asynchronously by signal handlers, for example.
401 528 self.showtraceback()
402 529 else:
403 530 self.input_splitter.push(line)
404 531 more = self.input_splitter.push_accepts_more()
405 532 if (self.SyntaxTB.last_syntax_error and
406 533 self.autoedit_syntax):
407 534 self.edit_syntax_error()
408 535 if not more:
409 536 source_raw = self.input_splitter.source_raw_reset()[1]
410 537 self.run_cell(source_raw, store_history=True)
411 538 hlen_b4_cell = \
412 539 self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
413 540
414 541 # Turn off the exit flag, so the mainloop can be restarted if desired
415 542 self.exit_now = False
416 543
417 544 def raw_input(self, prompt=''):
418 545 """Write a prompt and read a line.
419 546
420 547 The returned line does not include the trailing newline.
421 548 When the user enters the EOF key sequence, EOFError is raised.
422 549
423 550 Optional inputs:
424 551
425 552 - prompt(''): a string to be printed to prompt the user.
426 553
427 554 - continue_prompt(False): whether this line is the first one or a
428 555 continuation in a sequence of inputs.
429 556 """
430 557 # Code run by the user may have modified the readline completer state.
431 558 # We must ensure that our completer is back in place.
432 559
433 560 if self.has_readline:
434 561 self.set_readline_completer()
435 562
436 563 try:
437 564 line = py3compat.str_to_unicode(self.raw_input_original(prompt))
438 565 except ValueError:
439 566 warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
440 567 " or sys.stdout.close()!\nExiting IPython!")
441 568 self.ask_exit()
442 569 return ""
443 570
444 571 # Try to be reasonably smart about not re-indenting pasted input more
445 572 # than necessary. We do this by trimming out the auto-indent initial
446 573 # spaces, if the user's actual input started itself with whitespace.
447 574 if self.autoindent:
448 575 if num_ini_spaces(line) > self.indent_current_nsp:
449 576 line = line[self.indent_current_nsp:]
450 577 self.indent_current_nsp = 0
451 578
452 579 return line
453 580
454 581 #-------------------------------------------------------------------------
455 582 # Methods to support auto-editing of SyntaxErrors.
456 583 #-------------------------------------------------------------------------
457 584
458 585 def edit_syntax_error(self):
459 586 """The bottom half of the syntax error handler called in the main loop.
460 587
461 588 Loop until syntax error is fixed or user cancels.
462 589 """
463 590
464 591 while self.SyntaxTB.last_syntax_error:
465 592 # copy and clear last_syntax_error
466 593 err = self.SyntaxTB.clear_err_state()
467 594 if not self._should_recompile(err):
468 595 return
469 596 try:
470 597 # may set last_syntax_error again if a SyntaxError is raised
471 598 self.safe_execfile(err.filename,self.user_ns)
472 599 except:
473 600 self.showtraceback()
474 601 else:
475 602 try:
476 603 f = open(err.filename)
477 604 try:
478 605 # This should be inside a display_trap block and I
479 606 # think it is.
480 607 sys.displayhook(f.read())
481 608 finally:
482 609 f.close()
483 610 except:
484 611 self.showtraceback()
485 612
486 613 def _should_recompile(self,e):
487 614 """Utility routine for edit_syntax_error"""
488 615
489 616 if e.filename in ('<ipython console>','<input>','<string>',
490 617 '<console>','<BackgroundJob compilation>',
491 618 None):
492 619
493 620 return False
494 621 try:
495 622 if (self.autoedit_syntax and
496 623 not self.ask_yes_no('Return to editor to correct syntax error? '
497 624 '[Y/n] ','y')):
498 625 return False
499 626 except EOFError:
500 627 return False
501 628
502 629 def int0(x):
503 630 try:
504 631 return int(x)
505 632 except TypeError:
506 633 return 0
507 634 # always pass integer line and offset values to editor hook
508 635 try:
509 636 self.hooks.fix_error_editor(e.filename,
510 637 int0(e.lineno),int0(e.offset),e.msg)
511 638 except TryNext:
512 639 warn('Could not open editor')
513 640 return False
514 641 return True
515 642
516 643 #-------------------------------------------------------------------------
517 644 # Things related to exiting
518 645 #-------------------------------------------------------------------------
519 646
520 647 def ask_exit(self):
521 648 """ Ask the shell to exit. Can be overiden and used as a callback. """
522 649 self.exit_now = True
523 650
524 651 def exit(self):
525 652 """Handle interactive exit.
526 653
527 654 This method calls the ask_exit callback."""
528 655 if self.confirm_exit:
529 656 if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
530 657 self.ask_exit()
531 658 else:
532 659 self.ask_exit()
533 660
534 #------------------------------------------------------------------------
535 # Magic overrides
536 #------------------------------------------------------------------------
537 # Once the base class stops inheriting from magic, this code needs to be
538 # moved into a separate machinery as well. For now, at least isolate here
539 # the magics which this class needs to implement differently from the base
540 # class, or that are unique to it.
541
542 def magic_autoindent(self, parameter_s = ''):
543 """Toggle autoindent on/off (if available)."""
544
545 self.shell.set_autoindent()
546 print "Automatic indentation is:",['OFF','ON'][self.shell.autoindent]
547
548 @skip_doctest
549 def magic_cpaste(self, parameter_s=''):
550 """Paste & execute a pre-formatted code block from clipboard.
551
552 You must terminate the block with '--' (two minus-signs) or Ctrl-D
553 alone on the line. You can also provide your own sentinel with '%paste
554 -s %%' ('%%' is the new sentinel for this operation)
555
556 The block is dedented prior to execution to enable execution of method
557 definitions. '>' and '+' characters at the beginning of a line are
558 ignored, to allow pasting directly from e-mails, diff files and
559 doctests (the '...' continuation prompt is also stripped). The
560 executed block is also assigned to variable named 'pasted_block' for
561 later editing with '%edit pasted_block'.
562
563 You can also pass a variable name as an argument, e.g. '%cpaste foo'.
564 This assigns the pasted block to variable 'foo' as string, without
565 dedenting or executing it (preceding >>> and + is still stripped)
566
567 '%cpaste -r' re-executes the block previously entered by cpaste.
568
569 Do not be alarmed by garbled output on Windows (it's a readline bug).
570 Just press enter and type -- (and press enter again) and the block
571 will be what was just pasted.
572
573 IPython statements (magics, shell escapes) are not supported (yet).
574
575 See also
576 --------
577 paste: automatically pull code from clipboard.
578
579 Examples
580 --------
581 ::
582
583 In [8]: %cpaste
584 Pasting code; enter '--' alone on the line to stop.
585 :>>> a = ["world!", "Hello"]
586 :>>> print " ".join(sorted(a))
587 :--
588 Hello world!
589 """
590
591 opts, name = self.parse_options(parameter_s, 'rs:', mode='string')
592 if 'r' in opts:
593 rerun_pasted(self.shell)
594 return
595
596 sentinel = opts.get('s', '--')
597 block = strip_email_quotes(get_pasted_lines(sentinel))
598 store_or_execute(self.shell, block, name)
599
600 def magic_paste(self, parameter_s=''):
601 """Paste & execute a pre-formatted code block from clipboard.
602
603 The text is pulled directly from the clipboard without user
604 intervention and printed back on the screen before execution (unless
605 the -q flag is given to force quiet mode).
606
607 The block is dedented prior to execution to enable execution of method
608 definitions. '>' and '+' characters at the beginning of a line are
609 ignored, to allow pasting directly from e-mails, diff files and
610 doctests (the '...' continuation prompt is also stripped). The
611 executed block is also assigned to variable named 'pasted_block' for
612 later editing with '%edit pasted_block'.
613
614 You can also pass a variable name as an argument, e.g. '%paste foo'.
615 This assigns the pasted block to variable 'foo' as string, without
616 dedenting or executing it (preceding >>> and + is still stripped)
617
618 Options
619 -------
620
621 -r: re-executes the block previously entered by cpaste.
622
623 -q: quiet mode: do not echo the pasted text back to the terminal.
624
625 IPython statements (magics, shell escapes) are not supported (yet).
661 #-------------------------------------------------------------------------
662 # Things related to magics
663 #-------------------------------------------------------------------------
626 664
627 See also
628 --------
629 cpaste: manually paste code into terminal until you mark its end.
630 """
631 opts, name = self.parse_options(parameter_s, 'rq', mode='string')
632 if 'r' in opts:
633 rerun_pasted(self.shell)
634 return
665 def init_magics(self):
666 super(TerminalInteractiveShell, self).init_magics()
667 self.define_magic('autoindent', magic_autoindent)
668 self.define_magic('cpaste', magic_cpaste)
669 self.define_magic('paste', magic_paste)
635 670 try:
636 text = self.shell.hooks.clipboard_get()
637 block = strip_email_quotes(text.splitlines())
638 except TryNext as clipboard_exc:
639 message = getattr(clipboard_exc, 'args')
640 if message:
641 error(message[0])
671 magic_cls
672 except NameError:
673 pass
642 674 else:
643 error('Could not get text from the clipboard.')
644 return
645
646 # By default, echo back to terminal unless quiet mode is requested
647 if 'q' not in opts:
648 write = self.shell.write
649 write(self.shell.pycolorize(block))
650 if not block.endswith('\n'):
651 write('\n')
652 write("## -- End pasted text --\n")
653
654 store_or_execute(self.shell, block, name)
655
656 # Class-level: add a '%cls' magic only on Windows
657 if sys.platform == 'win32':
658 def magic_cls(self, s):
659 """Clear screen.
660 """
661 os.system("cls")
675 self.define_magic('cls', magic_cls)
662 676
663 677 def showindentationerror(self):
664 678 super(TerminalInteractiveShell, self).showindentationerror()
665 679 print("If you want to paste code into IPython, try the "
666 680 "%paste and %cpaste magic functions.")
667 681
668 682
669 683 InteractiveShellABC.register(TerminalInteractiveShell)
@@ -1,1100 +1,1100 b''
1 1 """Views of remote engines.
2 2
3 3 Authors:
4 4
5 5 * Min RK
6 6 """
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2010-2011 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import imp
19 19 import sys
20 20 import warnings
21 21 from contextlib import contextmanager
22 22 from types import ModuleType
23 23
24 24 import zmq
25 25
26 26 from IPython.testing.skipdoctest import skip_doctest
27 27 from IPython.utils.traitlets import (
28 28 HasTraits, Any, Bool, List, Dict, Set, Instance, CFloat, Integer
29 29 )
30 30 from IPython.external.decorator import decorator
31 31
32 32 from IPython.parallel import util
33 33 from IPython.parallel.controller.dependency import Dependency, dependent
34 34
35 35 from . import map as Map
36 36 from .asyncresult import AsyncResult, AsyncMapResult
37 37 from .remotefunction import ParallelFunction, parallel, remote, getname
38 38
39 39 #-----------------------------------------------------------------------------
40 40 # Decorators
41 41 #-----------------------------------------------------------------------------
42 42
43 43 @decorator
44 44 def save_ids(f, self, *args, **kwargs):
45 45 """Keep our history and outstanding attributes up to date after a method call."""
46 46 n_previous = len(self.client.history)
47 47 try:
48 48 ret = f(self, *args, **kwargs)
49 49 finally:
50 50 nmsgs = len(self.client.history) - n_previous
51 51 msg_ids = self.client.history[-nmsgs:]
52 52 self.history.extend(msg_ids)
53 53 map(self.outstanding.add, msg_ids)
54 54 return ret
55 55
56 56 @decorator
57 57 def sync_results(f, self, *args, **kwargs):
58 58 """sync relevant results from self.client to our results attribute."""
59 59 ret = f(self, *args, **kwargs)
60 60 delta = self.outstanding.difference(self.client.outstanding)
61 61 completed = self.outstanding.intersection(delta)
62 62 self.outstanding = self.outstanding.difference(completed)
63 63 for msg_id in completed:
64 64 self.results[msg_id] = self.client.results[msg_id]
65 65 return ret
66 66
67 67 @decorator
68 68 def spin_after(f, self, *args, **kwargs):
69 69 """call spin after the method."""
70 70 ret = f(self, *args, **kwargs)
71 71 self.spin()
72 72 return ret
73 73
74 74 #-----------------------------------------------------------------------------
75 75 # Classes
76 76 #-----------------------------------------------------------------------------
77 77
78 78 @skip_doctest
79 79 class View(HasTraits):
80 80 """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
81 81
82 82 Don't use this class, use subclasses.
83 83
84 84 Methods
85 85 -------
86 86
87 87 spin
88 88 flushes incoming results and registration state changes
89 89 control methods spin, and requesting `ids` also ensures up to date
90 90
91 91 wait
92 92 wait on one or more msg_ids
93 93
94 94 execution methods
95 95 apply
96 96 legacy: execute, run
97 97
98 98 data movement
99 99 push, pull, scatter, gather
100 100
101 101 query methods
102 102 get_result, queue_status, purge_results, result_status
103 103
104 104 control methods
105 105 abort, shutdown
106 106
107 107 """
108 108 # flags
109 109 block=Bool(False)
110 110 track=Bool(True)
111 111 targets = Any()
112 112
113 113 history=List()
114 114 outstanding = Set()
115 115 results = Dict()
116 116 client = Instance('IPython.parallel.Client')
117 117
118 118 _socket = Instance('zmq.Socket')
119 119 _flag_names = List(['targets', 'block', 'track'])
120 120 _targets = Any()
121 121 _idents = Any()
122 122
123 123 def __init__(self, client=None, socket=None, **flags):
124 124 super(View, self).__init__(client=client, _socket=socket)
125 125 self.block = client.block
126 126
127 127 self.set_flags(**flags)
128 128
129 129 assert not self.__class__ is View, "Don't use base View objects, use subclasses"
130 130
131 131 def __repr__(self):
132 132 strtargets = str(self.targets)
133 133 if len(strtargets) > 16:
134 134 strtargets = strtargets[:12]+'...]'
135 135 return "<%s %s>"%(self.__class__.__name__, strtargets)
136 136
137 137 def __len__(self):
138 138 if isinstance(self.targets, list):
139 139 return len(self.targets)
140 140 elif isinstance(self.targets, int):
141 141 return 1
142 142 else:
143 143 return len(self.client)
144 144
145 145 def set_flags(self, **kwargs):
146 146 """set my attribute flags by keyword.
147 147
148 148 Views determine behavior with a few attributes (`block`, `track`, etc.).
149 149 These attributes can be set all at once by name with this method.
150 150
151 151 Parameters
152 152 ----------
153 153
154 154 block : bool
155 155 whether to wait for results
156 156 track : bool
157 157 whether to create a MessageTracker to allow the user to
158 158 safely edit after arrays and buffers during non-copying
159 159 sends.
160 160 """
161 161 for name, value in kwargs.iteritems():
162 162 if name not in self._flag_names:
163 163 raise KeyError("Invalid name: %r"%name)
164 164 else:
165 165 setattr(self, name, value)
166 166
167 167 @contextmanager
168 168 def temp_flags(self, **kwargs):
169 169 """temporarily set flags, for use in `with` statements.
170 170
171 171 See set_flags for permanent setting of flags
172 172
173 173 Examples
174 174 --------
175 175
176 176 >>> view.track=False
177 177 ...
178 178 >>> with view.temp_flags(track=True):
179 179 ... ar = view.apply(dostuff, my_big_array)
180 180 ... ar.tracker.wait() # wait for send to finish
181 181 >>> view.track
182 182 False
183 183
184 184 """
185 185 # preflight: save flags, and set temporaries
186 186 saved_flags = {}
187 187 for f in self._flag_names:
188 188 saved_flags[f] = getattr(self, f)
189 189 self.set_flags(**kwargs)
190 190 # yield to the with-statement block
191 191 try:
192 192 yield
193 193 finally:
194 194 # postflight: restore saved flags
195 195 self.set_flags(**saved_flags)
196 196
197 197
198 198 #----------------------------------------------------------------
199 199 # apply
200 200 #----------------------------------------------------------------
201 201
202 202 @sync_results
203 203 @save_ids
204 204 def _really_apply(self, f, args, kwargs, block=None, **options):
205 205 """wrapper for client.send_apply_request"""
206 206 raise NotImplementedError("Implement in subclasses")
207 207
208 208 def apply(self, f, *args, **kwargs):
209 209 """calls f(*args, **kwargs) on remote engines, returning the result.
210 210
211 211 This method sets all apply flags via this View's attributes.
212 212
213 213 if self.block is False:
214 214 returns AsyncResult
215 215 else:
216 216 returns actual result of f(*args, **kwargs)
217 217 """
218 218 return self._really_apply(f, args, kwargs)
219 219
220 220 def apply_async(self, f, *args, **kwargs):
221 221 """calls f(*args, **kwargs) on remote engines in a nonblocking manner.
222 222
223 223 returns AsyncResult
224 224 """
225 225 return self._really_apply(f, args, kwargs, block=False)
226 226
227 227 @spin_after
228 228 def apply_sync(self, f, *args, **kwargs):
229 229 """calls f(*args, **kwargs) on remote engines in a blocking manner,
230 230 returning the result.
231 231
232 232 returns: actual result of f(*args, **kwargs)
233 233 """
234 234 return self._really_apply(f, args, kwargs, block=True)
235 235
236 236 #----------------------------------------------------------------
237 237 # wrappers for client and control methods
238 238 #----------------------------------------------------------------
239 239 @sync_results
240 240 def spin(self):
241 241 """spin the client, and sync"""
242 242 self.client.spin()
243 243
244 244 @sync_results
245 245 def wait(self, jobs=None, timeout=-1):
246 246 """waits on one or more `jobs`, for up to `timeout` seconds.
247 247
248 248 Parameters
249 249 ----------
250 250
251 251 jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
252 252 ints are indices to self.history
253 253 strs are msg_ids
254 254 default: wait on all outstanding messages
255 255 timeout : float
256 256 a time in seconds, after which to give up.
257 257 default is -1, which means no timeout
258 258
259 259 Returns
260 260 -------
261 261
262 262 True : when all msg_ids are done
263 263 False : timeout reached, some msg_ids still outstanding
264 264 """
265 265 if jobs is None:
266 266 jobs = self.history
267 267 return self.client.wait(jobs, timeout)
268 268
269 269 def abort(self, jobs=None, targets=None, block=None):
270 270 """Abort jobs on my engines.
271 271
272 272 Parameters
273 273 ----------
274 274
275 275 jobs : None, str, list of strs, optional
276 276 if None: abort all jobs.
277 277 else: abort specific msg_id(s).
278 278 """
279 279 block = block if block is not None else self.block
280 280 targets = targets if targets is not None else self.targets
281 281 jobs = jobs if jobs is not None else list(self.outstanding)
282 282
283 283 return self.client.abort(jobs=jobs, targets=targets, block=block)
284 284
285 285 def queue_status(self, targets=None, verbose=False):
286 286 """Fetch the Queue status of my engines"""
287 287 targets = targets if targets is not None else self.targets
288 288 return self.client.queue_status(targets=targets, verbose=verbose)
289 289
290 290 def purge_results(self, jobs=[], targets=[]):
291 291 """Instruct the controller to forget specific results."""
292 292 if targets is None or targets == 'all':
293 293 targets = self.targets
294 294 return self.client.purge_results(jobs=jobs, targets=targets)
295 295
296 296 def shutdown(self, targets=None, restart=False, hub=False, block=None):
297 297 """Terminates one or more engine processes, optionally including the hub.
298 298 """
299 299 block = self.block if block is None else block
300 300 if targets is None or targets == 'all':
301 301 targets = self.targets
302 302 return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block)
303 303
304 304 @spin_after
305 305 def get_result(self, indices_or_msg_ids=None):
306 306 """return one or more results, specified by history index or msg_id.
307 307
308 308 See client.get_result for details.
309 309
310 310 """
311 311
312 312 if indices_or_msg_ids is None:
313 313 indices_or_msg_ids = -1
314 314 if isinstance(indices_or_msg_ids, int):
315 315 indices_or_msg_ids = self.history[indices_or_msg_ids]
316 316 elif isinstance(indices_or_msg_ids, (list,tuple,set)):
317 317 indices_or_msg_ids = list(indices_or_msg_ids)
318 318 for i,index in enumerate(indices_or_msg_ids):
319 319 if isinstance(index, int):
320 320 indices_or_msg_ids[i] = self.history[index]
321 321 return self.client.get_result(indices_or_msg_ids)
322 322
323 323 #-------------------------------------------------------------------
324 324 # Map
325 325 #-------------------------------------------------------------------
326 326
327 327 def map(self, f, *sequences, **kwargs):
328 328 """override in subclasses"""
329 329 raise NotImplementedError
330 330
331 331 def map_async(self, f, *sequences, **kwargs):
332 332 """Parallel version of builtin `map`, using this view's engines.
333 333
334 334 This is equivalent to map(...block=False)
335 335
336 336 See `self.map` for details.
337 337 """
338 338 if 'block' in kwargs:
339 339 raise TypeError("map_async doesn't take a `block` keyword argument.")
340 340 kwargs['block'] = False
341 341 return self.map(f,*sequences,**kwargs)
342 342
343 343 def map_sync(self, f, *sequences, **kwargs):
344 344 """Parallel version of builtin `map`, using this view's engines.
345 345
346 346 This is equivalent to map(...block=True)
347 347
348 348 See `self.map` for details.
349 349 """
350 350 if 'block' in kwargs:
351 351 raise TypeError("map_sync doesn't take a `block` keyword argument.")
352 352 kwargs['block'] = True
353 353 return self.map(f,*sequences,**kwargs)
354 354
355 355 def imap(self, f, *sequences, **kwargs):
356 356 """Parallel version of `itertools.imap`.
357 357
358 358 See `self.map` for details.
359 359
360 360 """
361 361
362 362 return iter(self.map_async(f,*sequences, **kwargs))
363 363
364 364 #-------------------------------------------------------------------
365 365 # Decorators
366 366 #-------------------------------------------------------------------
367 367
368 368 def remote(self, block=True, **flags):
369 369 """Decorator for making a RemoteFunction"""
370 370 block = self.block if block is None else block
371 371 return remote(self, block=block, **flags)
372 372
373 373 def parallel(self, dist='b', block=None, **flags):
374 374 """Decorator for making a ParallelFunction"""
375 375 block = self.block if block is None else block
376 376 return parallel(self, dist=dist, block=block, **flags)
377 377
378 378 @skip_doctest
379 379 class DirectView(View):
380 380 """Direct Multiplexer View of one or more engines.
381 381
382 382 These are created via indexed access to a client:
383 383
384 384 >>> dv_1 = client[1]
385 385 >>> dv_all = client[:]
386 386 >>> dv_even = client[::2]
387 387 >>> dv_some = client[1:3]
388 388
389 389 This object provides dictionary access to engine namespaces:
390 390
391 391 # push a=5:
392 392 >>> dv['a'] = 5
393 393 # pull 'foo':
394 394 >>> db['foo']
395 395
396 396 """
397 397
398 398 def __init__(self, client=None, socket=None, targets=None):
399 399 super(DirectView, self).__init__(client=client, socket=socket, targets=targets)
400 400
401 401 @property
402 402 def importer(self):
403 403 """sync_imports(local=True) as a property.
404 404
405 405 See sync_imports for details.
406 406
407 407 """
408 408 return self.sync_imports(True)
409 409
410 410 @contextmanager
411 411 def sync_imports(self, local=True, quiet=False):
412 412 """Context Manager for performing simultaneous local and remote imports.
413 413
414 414 'import x as y' will *not* work. The 'as y' part will simply be ignored.
415 415
416 416 If `local=True`, then the package will also be imported locally.
417 417
418 418 If `quiet=True`, no output will be produced when attempting remote
419 419 imports.
420 420
421 421 Note that remote-only (`local=False`) imports have not been implemented.
422 422
423 423 >>> with view.sync_imports():
424 424 ... from numpy import recarray
425 425 importing recarray from numpy on engine(s)
426 426
427 427 """
428 428 import __builtin__
429 429 local_import = __builtin__.__import__
430 430 modules = set()
431 431 results = []
432 432 @util.interactive
433 433 def remote_import(name, fromlist, level):
434 434 """the function to be passed to apply, that actually performs the import
435 435 on the engine, and loads up the user namespace.
436 436 """
437 437 import sys
438 438 user_ns = globals()
439 439 mod = __import__(name, fromlist=fromlist, level=level)
440 440 if fromlist:
441 441 for key in fromlist:
442 442 user_ns[key] = getattr(mod, key)
443 443 else:
444 444 user_ns[name] = sys.modules[name]
445 445
446 446 def view_import(name, globals={}, locals={}, fromlist=[], level=-1):
447 447 """the drop-in replacement for __import__, that optionally imports
448 448 locally as well.
449 449 """
450 450 # don't override nested imports
451 451 save_import = __builtin__.__import__
452 452 __builtin__.__import__ = local_import
453 453
454 454 if imp.lock_held():
455 455 # this is a side-effect import, don't do it remotely, or even
456 456 # ignore the local effects
457 457 return local_import(name, globals, locals, fromlist, level)
458 458
459 459 imp.acquire_lock()
460 460 if local:
461 461 mod = local_import(name, globals, locals, fromlist, level)
462 462 else:
463 463 raise NotImplementedError("remote-only imports not yet implemented")
464 464 imp.release_lock()
465 465
466 466 key = name+':'+','.join(fromlist or [])
467 467 if level == -1 and key not in modules:
468 468 modules.add(key)
469 469 if not quiet:
470 470 if fromlist:
471 471 print "importing %s from %s on engine(s)"%(','.join(fromlist), name)
472 472 else:
473 473 print "importing %s on engine(s)"%name
474 474 results.append(self.apply_async(remote_import, name, fromlist, level))
475 475 # restore override
476 476 __builtin__.__import__ = save_import
477 477
478 478 return mod
479 479
480 480 # override __import__
481 481 __builtin__.__import__ = view_import
482 482 try:
483 483 # enter the block
484 484 yield
485 485 except ImportError:
486 486 if local:
487 487 raise
488 488 else:
489 489 # ignore import errors if not doing local imports
490 490 pass
491 491 finally:
492 492 # always restore __import__
493 493 __builtin__.__import__ = local_import
494 494
495 495 for r in results:
496 496 # raise possible remote ImportErrors here
497 497 r.get()
498 498
499 499
500 500 @sync_results
501 501 @save_ids
502 502 def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
503 503 """calls f(*args, **kwargs) on remote engines, returning the result.
504 504
505 505 This method sets all of `apply`'s flags via this View's attributes.
506 506
507 507 Parameters
508 508 ----------
509 509
510 510 f : callable
511 511
512 512 args : list [default: empty]
513 513
514 514 kwargs : dict [default: empty]
515 515
516 516 targets : target list [default: self.targets]
517 517 where to run
518 518 block : bool [default: self.block]
519 519 whether to block
520 520 track : bool [default: self.track]
521 521 whether to ask zmq to track the message, for safe non-copying sends
522 522
523 523 Returns
524 524 -------
525 525
526 526 if self.block is False:
527 527 returns AsyncResult
528 528 else:
529 529 returns actual result of f(*args, **kwargs) on the engine(s)
530 530 This will be a list of self.targets is also a list (even length 1), or
531 531 the single result if self.targets is an integer engine id
532 532 """
533 533 args = [] if args is None else args
534 534 kwargs = {} if kwargs is None else kwargs
535 535 block = self.block if block is None else block
536 536 track = self.track if track is None else track
537 537 targets = self.targets if targets is None else targets
538 538
539 539 _idents = self.client._build_targets(targets)[0]
540 540 msg_ids = []
541 541 trackers = []
542 542 for ident in _idents:
543 543 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
544 544 ident=ident)
545 545 if track:
546 546 trackers.append(msg['tracker'])
547 547 msg_ids.append(msg['header']['msg_id'])
548 548 tracker = None if track is False else zmq.MessageTracker(*trackers)
549 549 ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=targets, tracker=tracker)
550 550 if block:
551 551 try:
552 552 return ar.get()
553 553 except KeyboardInterrupt:
554 554 pass
555 555 return ar
556 556
557 557
558 558 @spin_after
559 559 def map(self, f, *sequences, **kwargs):
560 560 """view.map(f, *sequences, block=self.block) => list|AsyncMapResult
561 561
562 562 Parallel version of builtin `map`, using this View's `targets`.
563 563
564 564 There will be one task per target, so work will be chunked
565 565 if the sequences are longer than `targets`.
566 566
567 567 Results can be iterated as they are ready, but will become available in chunks.
568 568
569 569 Parameters
570 570 ----------
571 571
572 572 f : callable
573 573 function to be mapped
574 574 *sequences: one or more sequences of matching length
575 575 the sequences to be distributed and passed to `f`
576 576 block : bool
577 577 whether to wait for the result or not [default self.block]
578 578
579 579 Returns
580 580 -------
581 581
582 582 if block=False:
583 583 AsyncMapResult
584 584 An object like AsyncResult, but which reassembles the sequence of results
585 585 into a single list. AsyncMapResults can be iterated through before all
586 586 results are complete.
587 587 else:
588 588 list
589 589 the result of map(f,*sequences)
590 590 """
591 591
592 592 block = kwargs.pop('block', self.block)
593 593 for k in kwargs.keys():
594 594 if k not in ['block', 'track']:
595 595 raise TypeError("invalid keyword arg, %r"%k)
596 596
597 597 assert len(sequences) > 0, "must have some sequences to map onto!"
598 598 pf = ParallelFunction(self, f, block=block, **kwargs)
599 599 return pf.map(*sequences)
600 600
601 601 @sync_results
602 602 @save_ids
603 603 def execute(self, code, silent=True, targets=None, block=None):
604 604 """Executes `code` on `targets` in blocking or nonblocking manner.
605 605
606 606 ``execute`` is always `bound` (affects engine namespace)
607 607
608 608 Parameters
609 609 ----------
610 610
611 611 code : str
612 612 the code string to be executed
613 613 block : bool
614 614 whether or not to wait until done to return
615 615 default: self.block
616 616 """
617 617 block = self.block if block is None else block
618 618 targets = self.targets if targets is None else targets
619 619
620 620 _idents = self.client._build_targets(targets)[0]
621 621 msg_ids = []
622 622 trackers = []
623 623 for ident in _idents:
624 624 msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident)
625 625 msg_ids.append(msg['header']['msg_id'])
626 626 ar = AsyncResult(self.client, msg_ids, fname='execute', targets=targets)
627 627 if block:
628 628 try:
629 629 ar.get()
630 630 except KeyboardInterrupt:
631 631 pass
632 632 return ar
633 633
634 634 def run(self, filename, targets=None, block=None):
635 635 """Execute contents of `filename` on my engine(s).
636 636
637 637 This simply reads the contents of the file and calls `execute`.
638 638
639 639 Parameters
640 640 ----------
641 641
642 642 filename : str
643 643 The path to the file
644 644 targets : int/str/list of ints/strs
645 645 the engines on which to execute
646 646 default : all
647 647 block : bool
648 648 whether or not to wait until done
649 649 default: self.block
650 650
651 651 """
652 652 with open(filename, 'r') as f:
653 653 # add newline in case of trailing indented whitespace
654 654 # which will cause SyntaxError
655 655 code = f.read()+'\n'
656 656 return self.execute(code, block=block, targets=targets)
657 657
658 658 def update(self, ns):
659 659 """update remote namespace with dict `ns`
660 660
661 661 See `push` for details.
662 662 """
663 663 return self.push(ns, block=self.block, track=self.track)
664 664
665 665 def push(self, ns, targets=None, block=None, track=None):
666 666 """update remote namespace with dict `ns`
667 667
668 668 Parameters
669 669 ----------
670 670
671 671 ns : dict
672 672 dict of keys with which to update engine namespace(s)
673 673 block : bool [default : self.block]
674 674 whether to wait to be notified of engine receipt
675 675
676 676 """
677 677
678 678 block = block if block is not None else self.block
679 679 track = track if track is not None else self.track
680 680 targets = targets if targets is not None else self.targets
681 681 # applier = self.apply_sync if block else self.apply_async
682 682 if not isinstance(ns, dict):
683 683 raise TypeError("Must be a dict, not %s"%type(ns))
684 684 return self._really_apply(util._push, kwargs=ns, block=block, track=track, targets=targets)
685 685
686 686 def get(self, key_s):
687 687 """get object(s) by `key_s` from remote namespace
688 688
689 689 see `pull` for details.
690 690 """
691 691 # block = block if block is not None else self.block
692 692 return self.pull(key_s, block=True)
693 693
694 694 def pull(self, names, targets=None, block=None):
695 695 """get object(s) by `name` from remote namespace
696 696
697 697 will return one object if it is a key.
698 698 can also take a list of keys, in which case it will return a list of objects.
699 699 """
700 700 block = block if block is not None else self.block
701 701 targets = targets if targets is not None else self.targets
702 702 applier = self.apply_sync if block else self.apply_async
703 703 if isinstance(names, basestring):
704 704 pass
705 705 elif isinstance(names, (list,tuple,set)):
706 706 for key in names:
707 707 if not isinstance(key, basestring):
708 708 raise TypeError("keys must be str, not type %r"%type(key))
709 709 else:
710 710 raise TypeError("names must be strs, not %r"%names)
711 711 return self._really_apply(util._pull, (names,), block=block, targets=targets)
712 712
713 713 def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
714 714 """
715 715 Partition a Python sequence and send the partitions to a set of engines.
716 716 """
717 717 block = block if block is not None else self.block
718 718 track = track if track is not None else self.track
719 719 targets = targets if targets is not None else self.targets
720 720
721 721 # construct integer ID list:
722 722 targets = self.client._build_targets(targets)[1]
723 723
724 724 mapObject = Map.dists[dist]()
725 725 nparts = len(targets)
726 726 msg_ids = []
727 727 trackers = []
728 728 for index, engineid in enumerate(targets):
729 729 partition = mapObject.getPartition(seq, index, nparts)
730 730 if flatten and len(partition) == 1:
731 731 ns = {key: partition[0]}
732 732 else:
733 733 ns = {key: partition}
734 734 r = self.push(ns, block=False, track=track, targets=engineid)
735 735 msg_ids.extend(r.msg_ids)
736 736 if track:
737 737 trackers.append(r._tracker)
738 738
739 739 if track:
740 740 tracker = zmq.MessageTracker(*trackers)
741 741 else:
742 742 tracker = None
743 743
744 744 r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
745 745 if block:
746 746 r.wait()
747 747 else:
748 748 return r
749 749
750 750 @sync_results
751 751 @save_ids
752 752 def gather(self, key, dist='b', targets=None, block=None):
753 753 """
754 754 Gather a partitioned sequence on a set of engines as a single local seq.
755 755 """
756 756 block = block if block is not None else self.block
757 757 targets = targets if targets is not None else self.targets
758 758 mapObject = Map.dists[dist]()
759 759 msg_ids = []
760 760
761 761 # construct integer ID list:
762 762 targets = self.client._build_targets(targets)[1]
763 763
764 764 for index, engineid in enumerate(targets):
765 765 msg_ids.extend(self.pull(key, block=False, targets=engineid).msg_ids)
766 766
767 767 r = AsyncMapResult(self.client, msg_ids, mapObject, fname='gather')
768 768
769 769 if block:
770 770 try:
771 771 return r.get()
772 772 except KeyboardInterrupt:
773 773 pass
774 774 return r
775 775
776 776 def __getitem__(self, key):
777 777 return self.get(key)
778 778
779 779 def __setitem__(self,key, value):
780 780 self.update({key:value})
781 781
782 782 def clear(self, targets=None, block=False):
783 783 """Clear the remote namespaces on my engines."""
784 784 block = block if block is not None else self.block
785 785 targets = targets if targets is not None else self.targets
786 786 return self.client.clear(targets=targets, block=block)
787 787
788 788 def kill(self, targets=None, block=True):
789 789 """Kill my engines."""
790 790 block = block if block is not None else self.block
791 791 targets = targets if targets is not None else self.targets
792 792 return self.client.kill(targets=targets, block=block)
793 793
794 794 #----------------------------------------
795 795 # activate for %px,%autopx magics
796 796 #----------------------------------------
797 797 def activate(self):
798 798 """Make this `View` active for parallel magic commands.
799 799
800 800 IPython has a magic command syntax to work with `MultiEngineClient` objects.
801 801 In a given IPython session there is a single active one. While
802 802 there can be many `Views` created and used by the user,
803 803 there is only one active one. The active `View` is used whenever
804 804 the magic commands %px and %autopx are used.
805 805
806 806 The activate() method is called on a given `View` to make it
807 807 active. Once this has been done, the magic commands can be used.
808 808 """
809 809
810 810 try:
811 811 # This is injected into __builtins__.
812 812 ip = get_ipython()
813 813 except NameError:
814 814 print "The IPython parallel magics (%result, %px, %autopx) only work within IPython."
815 815 else:
816 816 pmagic = ip.plugin_manager.get_plugin('parallelmagic')
817 817 if pmagic is None:
818 ip.magic_load_ext('parallelmagic')
818 ip.magic('load_ext parallelmagic')
819 819 pmagic = ip.plugin_manager.get_plugin('parallelmagic')
820 820
821 821 pmagic.active_view = self
822 822
823 823
824 824 @skip_doctest
825 825 class LoadBalancedView(View):
826 826 """An load-balancing View that only executes via the Task scheduler.
827 827
828 828 Load-balanced views can be created with the client's `view` method:
829 829
830 830 >>> v = client.load_balanced_view()
831 831
832 832 or targets can be specified, to restrict the potential destinations:
833 833
834 834 >>> v = client.client.load_balanced_view([1,3])
835 835
836 836 which would restrict loadbalancing to between engines 1 and 3.
837 837
838 838 """
839 839
840 840 follow=Any()
841 841 after=Any()
842 842 timeout=CFloat()
843 843 retries = Integer(0)
844 844
845 845 _task_scheme = Any()
846 846 _flag_names = List(['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
847 847
848 848 def __init__(self, client=None, socket=None, **flags):
849 849 super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags)
850 850 self._task_scheme=client._task_scheme
851 851
852 852 def _validate_dependency(self, dep):
853 853 """validate a dependency.
854 854
855 855 For use in `set_flags`.
856 856 """
857 857 if dep is None or isinstance(dep, (basestring, AsyncResult, Dependency)):
858 858 return True
859 859 elif isinstance(dep, (list,set, tuple)):
860 860 for d in dep:
861 861 if not isinstance(d, (basestring, AsyncResult)):
862 862 return False
863 863 elif isinstance(dep, dict):
864 864 if set(dep.keys()) != set(Dependency().as_dict().keys()):
865 865 return False
866 866 if not isinstance(dep['msg_ids'], list):
867 867 return False
868 868 for d in dep['msg_ids']:
869 869 if not isinstance(d, basestring):
870 870 return False
871 871 else:
872 872 return False
873 873
874 874 return True
875 875
876 876 def _render_dependency(self, dep):
877 877 """helper for building jsonable dependencies from various input forms."""
878 878 if isinstance(dep, Dependency):
879 879 return dep.as_dict()
880 880 elif isinstance(dep, AsyncResult):
881 881 return dep.msg_ids
882 882 elif dep is None:
883 883 return []
884 884 else:
885 885 # pass to Dependency constructor
886 886 return list(Dependency(dep))
887 887
888 888 def set_flags(self, **kwargs):
889 889 """set my attribute flags by keyword.
890 890
891 891 A View is a wrapper for the Client's apply method, but with attributes
892 892 that specify keyword arguments, those attributes can be set by keyword
893 893 argument with this method.
894 894
895 895 Parameters
896 896 ----------
897 897
898 898 block : bool
899 899 whether to wait for results
900 900 track : bool
901 901 whether to create a MessageTracker to allow the user to
902 902 safely edit after arrays and buffers during non-copying
903 903 sends.
904 904
905 905 after : Dependency or collection of msg_ids
906 906 Only for load-balanced execution (targets=None)
907 907 Specify a list of msg_ids as a time-based dependency.
908 908 This job will only be run *after* the dependencies
909 909 have been met.
910 910
911 911 follow : Dependency or collection of msg_ids
912 912 Only for load-balanced execution (targets=None)
913 913 Specify a list of msg_ids as a location-based dependency.
914 914 This job will only be run on an engine where this dependency
915 915 is met.
916 916
917 917 timeout : float/int or None
918 918 Only for load-balanced execution (targets=None)
919 919 Specify an amount of time (in seconds) for the scheduler to
920 920 wait for dependencies to be met before failing with a
921 921 DependencyTimeout.
922 922
923 923 retries : int
924 924 Number of times a task will be retried on failure.
925 925 """
926 926
927 927 super(LoadBalancedView, self).set_flags(**kwargs)
928 928 for name in ('follow', 'after'):
929 929 if name in kwargs:
930 930 value = kwargs[name]
931 931 if self._validate_dependency(value):
932 932 setattr(self, name, value)
933 933 else:
934 934 raise ValueError("Invalid dependency: %r"%value)
935 935 if 'timeout' in kwargs:
936 936 t = kwargs['timeout']
937 937 if not isinstance(t, (int, long, float, type(None))):
938 938 raise TypeError("Invalid type for timeout: %r"%type(t))
939 939 if t is not None:
940 940 if t < 0:
941 941 raise ValueError("Invalid timeout: %s"%t)
942 942 self.timeout = t
943 943
944 944 @sync_results
945 945 @save_ids
946 946 def _really_apply(self, f, args=None, kwargs=None, block=None, track=None,
947 947 after=None, follow=None, timeout=None,
948 948 targets=None, retries=None):
949 949 """calls f(*args, **kwargs) on a remote engine, returning the result.
950 950
951 951 This method temporarily sets all of `apply`'s flags for a single call.
952 952
953 953 Parameters
954 954 ----------
955 955
956 956 f : callable
957 957
958 958 args : list [default: empty]
959 959
960 960 kwargs : dict [default: empty]
961 961
962 962 block : bool [default: self.block]
963 963 whether to block
964 964 track : bool [default: self.track]
965 965 whether to ask zmq to track the message, for safe non-copying sends
966 966
967 967 !!!!!! TODO: THE REST HERE !!!!
968 968
969 969 Returns
970 970 -------
971 971
972 972 if self.block is False:
973 973 returns AsyncResult
974 974 else:
975 975 returns actual result of f(*args, **kwargs) on the engine(s)
976 976 This will be a list of self.targets is also a list (even length 1), or
977 977 the single result if self.targets is an integer engine id
978 978 """
979 979
980 980 # validate whether we can run
981 981 if self._socket.closed:
982 982 msg = "Task farming is disabled"
983 983 if self._task_scheme == 'pure':
984 984 msg += " because the pure ZMQ scheduler cannot handle"
985 985 msg += " disappearing engines."
986 986 raise RuntimeError(msg)
987 987
988 988 if self._task_scheme == 'pure':
989 989 # pure zmq scheme doesn't support extra features
990 990 msg = "Pure ZMQ scheduler doesn't support the following flags:"
991 991 "follow, after, retries, targets, timeout"
992 992 if (follow or after or retries or targets or timeout):
993 993 # hard fail on Scheduler flags
994 994 raise RuntimeError(msg)
995 995 if isinstance(f, dependent):
996 996 # soft warn on functional dependencies
997 997 warnings.warn(msg, RuntimeWarning)
998 998
999 999 # build args
1000 1000 args = [] if args is None else args
1001 1001 kwargs = {} if kwargs is None else kwargs
1002 1002 block = self.block if block is None else block
1003 1003 track = self.track if track is None else track
1004 1004 after = self.after if after is None else after
1005 1005 retries = self.retries if retries is None else retries
1006 1006 follow = self.follow if follow is None else follow
1007 1007 timeout = self.timeout if timeout is None else timeout
1008 1008 targets = self.targets if targets is None else targets
1009 1009
1010 1010 if not isinstance(retries, int):
1011 1011 raise TypeError('retries must be int, not %r'%type(retries))
1012 1012
1013 1013 if targets is None:
1014 1014 idents = []
1015 1015 else:
1016 1016 idents = self.client._build_targets(targets)[0]
1017 1017 # ensure *not* bytes
1018 1018 idents = [ ident.decode() for ident in idents ]
1019 1019
1020 1020 after = self._render_dependency(after)
1021 1021 follow = self._render_dependency(follow)
1022 1022 subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries)
1023 1023
1024 1024 msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
1025 1025 subheader=subheader)
1026 1026 tracker = None if track is False else msg['tracker']
1027 1027
1028 1028 ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker)
1029 1029
1030 1030 if block:
1031 1031 try:
1032 1032 return ar.get()
1033 1033 except KeyboardInterrupt:
1034 1034 pass
1035 1035 return ar
1036 1036
1037 1037 @spin_after
1038 1038 @save_ids
1039 1039 def map(self, f, *sequences, **kwargs):
1040 1040 """view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult
1041 1041
1042 1042 Parallel version of builtin `map`, load-balanced by this View.
1043 1043
1044 1044 `block`, and `chunksize` can be specified by keyword only.
1045 1045
1046 1046 Each `chunksize` elements will be a separate task, and will be
1047 1047 load-balanced. This lets individual elements be available for iteration
1048 1048 as soon as they arrive.
1049 1049
1050 1050 Parameters
1051 1051 ----------
1052 1052
1053 1053 f : callable
1054 1054 function to be mapped
1055 1055 *sequences: one or more sequences of matching length
1056 1056 the sequences to be distributed and passed to `f`
1057 1057 block : bool [default self.block]
1058 1058 whether to wait for the result or not
1059 1059 track : bool
1060 1060 whether to create a MessageTracker to allow the user to
1061 1061 safely edit after arrays and buffers during non-copying
1062 1062 sends.
1063 1063 chunksize : int [default 1]
1064 1064 how many elements should be in each task.
1065 1065 ordered : bool [default True]
1066 1066 Whether the results should be gathered as they arrive, or enforce
1067 1067 the order of submission.
1068 1068
1069 1069 Only applies when iterating through AsyncMapResult as results arrive.
1070 1070 Has no effect when block=True.
1071 1071
1072 1072 Returns
1073 1073 -------
1074 1074
1075 1075 if block=False:
1076 1076 AsyncMapResult
1077 1077 An object like AsyncResult, but which reassembles the sequence of results
1078 1078 into a single list. AsyncMapResults can be iterated through before all
1079 1079 results are complete.
1080 1080 else:
1081 1081 the result of map(f,*sequences)
1082 1082
1083 1083 """
1084 1084
1085 1085 # default
1086 1086 block = kwargs.get('block', self.block)
1087 1087 chunksize = kwargs.get('chunksize', 1)
1088 1088 ordered = kwargs.get('ordered', True)
1089 1089
1090 1090 keyset = set(kwargs.keys())
1091 1091 extra_keys = keyset.difference_update(set(['block', 'chunksize']))
1092 1092 if extra_keys:
1093 1093 raise TypeError("Invalid kwargs: %s"%list(extra_keys))
1094 1094
1095 1095 assert len(sequences) > 0, "must have some sequences to map onto!"
1096 1096
1097 1097 pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered)
1098 1098 return pf.map(*sequences)
1099 1099
1100 1100 __all__ = ['LoadBalancedView', 'DirectView']
@@ -1,694 +1,694 b''
1 1 # -*- coding: utf-8 -*-
2 2 """test View objects
3 3
4 4 Authors:
5 5
6 6 * Min RK
7 7 """
8 8 #-------------------------------------------------------------------------------
9 9 # Copyright (C) 2011 The IPython Development Team
10 10 #
11 11 # Distributed under the terms of the BSD License. The full license is in
12 12 # the file COPYING, distributed as part of this software.
13 13 #-------------------------------------------------------------------------------
14 14
15 15 #-------------------------------------------------------------------------------
16 16 # Imports
17 17 #-------------------------------------------------------------------------------
18 18
19 19 import sys
20 20 import time
21 21 from tempfile import mktemp
22 22 from StringIO import StringIO
23 23
24 24 import zmq
25 25 from nose import SkipTest
26 26
27 27 from IPython.testing import decorators as dec
28 28 from IPython.testing.ipunittest import ParametricTestCase
29 29
30 30 from IPython import parallel as pmod
31 31 from IPython.parallel import error
32 32 from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult
33 33 from IPython.parallel import DirectView
34 34 from IPython.parallel.util import interactive
35 35
36 36 from IPython.parallel.tests import add_engines
37 37
38 38 from .clienttest import ClusterTestCase, crash, wait, skip_without
39 39
40 40 def setup():
41 41 add_engines(3, total=True)
42 42
43 43 class TestView(ClusterTestCase, ParametricTestCase):
44 44
45 45 def test_z_crash_mux(self):
46 46 """test graceful handling of engine death (direct)"""
47 47 raise SkipTest("crash tests disabled, due to undesirable crash reports")
48 48 # self.add_engines(1)
49 49 eid = self.client.ids[-1]
50 50 ar = self.client[eid].apply_async(crash)
51 51 self.assertRaisesRemote(error.EngineError, ar.get, 10)
52 52 eid = ar.engine_id
53 53 tic = time.time()
54 54 while eid in self.client.ids and time.time()-tic < 5:
55 55 time.sleep(.01)
56 56 self.client.spin()
57 57 self.assertFalse(eid in self.client.ids, "Engine should have died")
58 58
59 59 def test_push_pull(self):
60 60 """test pushing and pulling"""
61 61 data = dict(a=10, b=1.05, c=range(10), d={'e':(1,2),'f':'hi'})
62 62 t = self.client.ids[-1]
63 63 v = self.client[t]
64 64 push = v.push
65 65 pull = v.pull
66 66 v.block=True
67 67 nengines = len(self.client)
68 68 push({'data':data})
69 69 d = pull('data')
70 70 self.assertEquals(d, data)
71 71 self.client[:].push({'data':data})
72 72 d = self.client[:].pull('data', block=True)
73 73 self.assertEquals(d, nengines*[data])
74 74 ar = push({'data':data}, block=False)
75 75 self.assertTrue(isinstance(ar, AsyncResult))
76 76 r = ar.get()
77 77 ar = self.client[:].pull('data', block=False)
78 78 self.assertTrue(isinstance(ar, AsyncResult))
79 79 r = ar.get()
80 80 self.assertEquals(r, nengines*[data])
81 81 self.client[:].push(dict(a=10,b=20))
82 82 r = self.client[:].pull(('a','b'), block=True)
83 83 self.assertEquals(r, nengines*[[10,20]])
84 84
85 85 def test_push_pull_function(self):
86 86 "test pushing and pulling functions"
87 87 def testf(x):
88 88 return 2.0*x
89 89
90 90 t = self.client.ids[-1]
91 91 v = self.client[t]
92 92 v.block=True
93 93 push = v.push
94 94 pull = v.pull
95 95 execute = v.execute
96 96 push({'testf':testf})
97 97 r = pull('testf')
98 98 self.assertEqual(r(1.0), testf(1.0))
99 99 execute('r = testf(10)')
100 100 r = pull('r')
101 101 self.assertEquals(r, testf(10))
102 102 ar = self.client[:].push({'testf':testf}, block=False)
103 103 ar.get()
104 104 ar = self.client[:].pull('testf', block=False)
105 105 rlist = ar.get()
106 106 for r in rlist:
107 107 self.assertEqual(r(1.0), testf(1.0))
108 108 execute("def g(x): return x*x")
109 109 r = pull(('testf','g'))
110 110 self.assertEquals((r[0](10),r[1](10)), (testf(10), 100))
111 111
112 112 def test_push_function_globals(self):
113 113 """test that pushed functions have access to globals"""
114 114 @interactive
115 115 def geta():
116 116 return a
117 117 # self.add_engines(1)
118 118 v = self.client[-1]
119 119 v.block=True
120 120 v['f'] = geta
121 121 self.assertRaisesRemote(NameError, v.execute, 'b=f()')
122 122 v.execute('a=5')
123 123 v.execute('b=f()')
124 124 self.assertEquals(v['b'], 5)
125 125
126 126 def test_push_function_defaults(self):
127 127 """test that pushed functions preserve default args"""
128 128 def echo(a=10):
129 129 return a
130 130 v = self.client[-1]
131 131 v.block=True
132 132 v['f'] = echo
133 133 v.execute('b=f()')
134 134 self.assertEquals(v['b'], 10)
135 135
136 136 def test_get_result(self):
137 137 """test getting results from the Hub."""
138 138 c = pmod.Client(profile='iptest')
139 139 # self.add_engines(1)
140 140 t = c.ids[-1]
141 141 v = c[t]
142 142 v2 = self.client[t]
143 143 ar = v.apply_async(wait, 1)
144 144 # give the monitor time to notice the message
145 145 time.sleep(.25)
146 146 ahr = v2.get_result(ar.msg_ids)
147 147 self.assertTrue(isinstance(ahr, AsyncHubResult))
148 148 self.assertEquals(ahr.get(), ar.get())
149 149 ar2 = v2.get_result(ar.msg_ids)
150 150 self.assertFalse(isinstance(ar2, AsyncHubResult))
151 151 c.spin()
152 152 c.close()
153 153
154 154 def test_run_newline(self):
155 155 """test that run appends newline to files"""
156 156 tmpfile = mktemp()
157 157 with open(tmpfile, 'w') as f:
158 158 f.write("""def g():
159 159 return 5
160 160 """)
161 161 v = self.client[-1]
162 162 v.run(tmpfile, block=True)
163 163 self.assertEquals(v.apply_sync(lambda f: f(), pmod.Reference('g')), 5)
164 164
165 165 def test_apply_tracked(self):
166 166 """test tracking for apply"""
167 167 # self.add_engines(1)
168 168 t = self.client.ids[-1]
169 169 v = self.client[t]
170 170 v.block=False
171 171 def echo(n=1024*1024, **kwargs):
172 172 with v.temp_flags(**kwargs):
173 173 return v.apply(lambda x: x, 'x'*n)
174 174 ar = echo(1, track=False)
175 175 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
176 176 self.assertTrue(ar.sent)
177 177 ar = echo(track=True)
178 178 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
179 179 self.assertEquals(ar.sent, ar._tracker.done)
180 180 ar._tracker.wait()
181 181 self.assertTrue(ar.sent)
182 182
183 183 def test_push_tracked(self):
184 184 t = self.client.ids[-1]
185 185 ns = dict(x='x'*1024*1024)
186 186 v = self.client[t]
187 187 ar = v.push(ns, block=False, track=False)
188 188 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
189 189 self.assertTrue(ar.sent)
190 190
191 191 ar = v.push(ns, block=False, track=True)
192 192 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
193 193 ar._tracker.wait()
194 194 self.assertEquals(ar.sent, ar._tracker.done)
195 195 self.assertTrue(ar.sent)
196 196 ar.get()
197 197
198 198 def test_scatter_tracked(self):
199 199 t = self.client.ids
200 200 x='x'*1024*1024
201 201 ar = self.client[t].scatter('x', x, block=False, track=False)
202 202 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
203 203 self.assertTrue(ar.sent)
204 204
205 205 ar = self.client[t].scatter('x', x, block=False, track=True)
206 206 self.assertTrue(isinstance(ar._tracker, zmq.MessageTracker))
207 207 self.assertEquals(ar.sent, ar._tracker.done)
208 208 ar._tracker.wait()
209 209 self.assertTrue(ar.sent)
210 210 ar.get()
211 211
212 212 def test_remote_reference(self):
213 213 v = self.client[-1]
214 214 v['a'] = 123
215 215 ra = pmod.Reference('a')
216 216 b = v.apply_sync(lambda x: x, ra)
217 217 self.assertEquals(b, 123)
218 218
219 219
220 220 def test_scatter_gather(self):
221 221 view = self.client[:]
222 222 seq1 = range(16)
223 223 view.scatter('a', seq1)
224 224 seq2 = view.gather('a', block=True)
225 225 self.assertEquals(seq2, seq1)
226 226 self.assertRaisesRemote(NameError, view.gather, 'asdf', block=True)
227 227
228 228 @skip_without('numpy')
229 229 def test_scatter_gather_numpy(self):
230 230 import numpy
231 231 from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
232 232 view = self.client[:]
233 233 a = numpy.arange(64)
234 234 view.scatter('a', a)
235 235 b = view.gather('a', block=True)
236 236 assert_array_equal(b, a)
237 237
238 238 def test_scatter_gather_lazy(self):
239 239 """scatter/gather with targets='all'"""
240 240 view = self.client.direct_view(targets='all')
241 241 x = range(64)
242 242 view.scatter('x', x)
243 243 gathered = view.gather('x', block=True)
244 244 self.assertEquals(gathered, x)
245 245
246 246
247 247 @dec.known_failure_py3
248 248 @skip_without('numpy')
249 249 def test_push_numpy_nocopy(self):
250 250 import numpy
251 251 view = self.client[:]
252 252 a = numpy.arange(64)
253 253 view['A'] = a
254 254 @interactive
255 255 def check_writeable(x):
256 256 return x.flags.writeable
257 257
258 258 for flag in view.apply_sync(check_writeable, pmod.Reference('A')):
259 259 self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
260 260
261 261 view.push(dict(B=a))
262 262 for flag in view.apply_sync(check_writeable, pmod.Reference('B')):
263 263 self.assertFalse(flag, "array is writeable, push shouldn't have pickled it")
264 264
265 265 @skip_without('numpy')
266 266 def test_apply_numpy(self):
267 267 """view.apply(f, ndarray)"""
268 268 import numpy
269 269 from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
270 270
271 271 A = numpy.random.random((100,100))
272 272 view = self.client[-1]
273 273 for dt in [ 'int32', 'uint8', 'float32', 'float64' ]:
274 274 B = A.astype(dt)
275 275 C = view.apply_sync(lambda x:x, B)
276 276 assert_array_equal(B,C)
277 277
278 278 def test_map(self):
279 279 view = self.client[:]
280 280 def f(x):
281 281 return x**2
282 282 data = range(16)
283 283 r = view.map_sync(f, data)
284 284 self.assertEquals(r, map(f, data))
285 285
286 286 def test_map_iterable(self):
287 287 """test map on iterables (direct)"""
288 288 view = self.client[:]
289 289 # 101 is prime, so it won't be evenly distributed
290 290 arr = range(101)
291 291 # ensure it will be an iterator, even in Python 3
292 292 it = iter(arr)
293 293 r = view.map_sync(lambda x:x, arr)
294 294 self.assertEquals(r, list(arr))
295 295
296 296 def test_scatterGatherNonblocking(self):
297 297 data = range(16)
298 298 view = self.client[:]
299 299 view.scatter('a', data, block=False)
300 300 ar = view.gather('a', block=False)
301 301 self.assertEquals(ar.get(), data)
302 302
303 303 @skip_without('numpy')
304 304 def test_scatter_gather_numpy_nonblocking(self):
305 305 import numpy
306 306 from numpy.testing.utils import assert_array_equal, assert_array_almost_equal
307 307 a = numpy.arange(64)
308 308 view = self.client[:]
309 309 ar = view.scatter('a', a, block=False)
310 310 self.assertTrue(isinstance(ar, AsyncResult))
311 311 amr = view.gather('a', block=False)
312 312 self.assertTrue(isinstance(amr, AsyncMapResult))
313 313 assert_array_equal(amr.get(), a)
314 314
315 315 def test_execute(self):
316 316 view = self.client[:]
317 317 # self.client.debug=True
318 318 execute = view.execute
319 319 ar = execute('c=30', block=False)
320 320 self.assertTrue(isinstance(ar, AsyncResult))
321 321 ar = execute('d=[0,1,2]', block=False)
322 322 self.client.wait(ar, 1)
323 323 self.assertEquals(len(ar.get()), len(self.client))
324 324 for c in view['c']:
325 325 self.assertEquals(c, 30)
326 326
327 327 def test_abort(self):
328 328 view = self.client[-1]
329 329 ar = view.execute('import time; time.sleep(1)', block=False)
330 330 ar2 = view.apply_async(lambda : 2)
331 331 ar3 = view.apply_async(lambda : 3)
332 332 view.abort(ar2)
333 333 view.abort(ar3.msg_ids)
334 334 self.assertRaises(error.TaskAborted, ar2.get)
335 335 self.assertRaises(error.TaskAborted, ar3.get)
336 336
337 337 def test_abort_all(self):
338 338 """view.abort() aborts all outstanding tasks"""
339 339 view = self.client[-1]
340 340 ars = [ view.apply_async(time.sleep, 0.25) for i in range(10) ]
341 341 view.abort()
342 342 view.wait(timeout=5)
343 343 for ar in ars[5:]:
344 344 self.assertRaises(error.TaskAborted, ar.get)
345 345
346 346 def test_temp_flags(self):
347 347 view = self.client[-1]
348 348 view.block=True
349 349 with view.temp_flags(block=False):
350 350 self.assertFalse(view.block)
351 351 self.assertTrue(view.block)
352 352
353 353 @dec.known_failure_py3
354 354 def test_importer(self):
355 355 view = self.client[-1]
356 356 view.clear(block=True)
357 357 with view.importer:
358 358 import re
359 359
360 360 @interactive
361 361 def findall(pat, s):
362 362 # this globals() step isn't necessary in real code
363 363 # only to prevent a closure in the test
364 364 re = globals()['re']
365 365 return re.findall(pat, s)
366 366
367 367 self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split())
368 368
369 369 # parallel magic tests
370 370
371 371 def test_magic_px_blocking(self):
372 372 ip = get_ipython()
373 373 v = self.client[-1]
374 374 v.activate()
375 375 v.block=True
376 376
377 ip.magic_px('a=5')
377 ip.magic('px a=5')
378 378 self.assertEquals(v['a'], 5)
379 ip.magic_px('a=10')
379 ip.magic('px a=10')
380 380 self.assertEquals(v['a'], 10)
381 381 sio = StringIO()
382 382 savestdout = sys.stdout
383 383 sys.stdout = sio
384 384 # just 'print a' worst ~99% of the time, but this ensures that
385 385 # the stdout message has arrived when the result is finished:
386 ip.magic_px('import sys,time;print (a); sys.stdout.flush();time.sleep(0.2)')
386 ip.magic('px import sys,time;print (a); sys.stdout.flush();time.sleep(0.2)')
387 387 sys.stdout = savestdout
388 388 buf = sio.getvalue()
389 389 self.assertTrue('[stdout:' in buf, buf)
390 390 self.assertTrue(buf.rstrip().endswith('10'))
391 self.assertRaisesRemote(ZeroDivisionError, ip.magic_px, '1/0')
391 self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0')
392 392
393 393 def test_magic_px_nonblocking(self):
394 394 ip = get_ipython()
395 395 v = self.client[-1]
396 396 v.activate()
397 397 v.block=False
398 398
399 ip.magic_px('a=5')
399 ip.magic('px a=5')
400 400 self.assertEquals(v['a'], 5)
401 ip.magic_px('a=10')
401 ip.magic('px a=10')
402 402 self.assertEquals(v['a'], 10)
403 403 sio = StringIO()
404 404 savestdout = sys.stdout
405 405 sys.stdout = sio
406 ip.magic_px('print a')
406 ip.magic('px print a')
407 407 sys.stdout = savestdout
408 408 buf = sio.getvalue()
409 409 self.assertFalse('[stdout:%i]'%v.targets in buf)
410 ip.magic_px('1/0')
410 ip.magic('px 1/0')
411 411 ar = v.get_result(-1)
412 412 self.assertRaisesRemote(ZeroDivisionError, ar.get)
413 413
414 414 def test_magic_autopx_blocking(self):
415 415 ip = get_ipython()
416 416 v = self.client[-1]
417 417 v.activate()
418 418 v.block=True
419 419
420 420 sio = StringIO()
421 421 savestdout = sys.stdout
422 422 sys.stdout = sio
423 ip.magic_autopx()
423 ip.magic('autopx')
424 424 ip.run_cell('\n'.join(('a=5','b=10','c=0')))
425 425 ip.run_cell('b*=2')
426 426 ip.run_cell('print (b)')
427 427 ip.run_cell("b/c")
428 ip.magic_autopx()
428 ip.magic('autopx')
429 429 sys.stdout = savestdout
430 430 output = sio.getvalue().strip()
431 431 self.assertTrue(output.startswith('%autopx enabled'))
432 432 self.assertTrue(output.endswith('%autopx disabled'))
433 433 self.assertTrue('RemoteError: ZeroDivisionError' in output)
434 434 ar = v.get_result(-1)
435 435 self.assertEquals(v['a'], 5)
436 436 self.assertEquals(v['b'], 20)
437 437 self.assertRaisesRemote(ZeroDivisionError, ar.get)
438 438
439 439 def test_magic_autopx_nonblocking(self):
440 440 ip = get_ipython()
441 441 v = self.client[-1]
442 442 v.activate()
443 443 v.block=False
444 444
445 445 sio = StringIO()
446 446 savestdout = sys.stdout
447 447 sys.stdout = sio
448 ip.magic_autopx()
448 ip.magic('autopx')
449 449 ip.run_cell('\n'.join(('a=5','b=10','c=0')))
450 450 ip.run_cell('print (b)')
451 451 ip.run_cell('import time; time.sleep(0.1)')
452 452 ip.run_cell("b/c")
453 453 ip.run_cell('b*=2')
454 ip.magic_autopx()
454 ip.magic('autopx')
455 455 sys.stdout = savestdout
456 456 output = sio.getvalue().strip()
457 457 self.assertTrue(output.startswith('%autopx enabled'))
458 458 self.assertTrue(output.endswith('%autopx disabled'))
459 459 self.assertFalse('ZeroDivisionError' in output)
460 460 ar = v.get_result(-2)
461 461 self.assertRaisesRemote(ZeroDivisionError, ar.get)
462 462 # prevent TaskAborted on pulls, due to ZeroDivisionError
463 463 time.sleep(0.5)
464 464 self.assertEquals(v['a'], 5)
465 465 # b*=2 will not fire, due to abort
466 466 self.assertEquals(v['b'], 10)
467 467
468 468 def test_magic_result(self):
469 469 ip = get_ipython()
470 470 v = self.client[-1]
471 471 v.activate()
472 472 v['a'] = 111
473 473 ra = v['a']
474 474
475 ar = ip.magic_result()
475 ar = ip.magic('result')
476 476 self.assertEquals(ar.msg_ids, [v.history[-1]])
477 477 self.assertEquals(ar.get(), 111)
478 ar = ip.magic_result('-2')
478 ar = ip.magic('result -2')
479 479 self.assertEquals(ar.msg_ids, [v.history[-2]])
480 480
481 481 def test_unicode_execute(self):
482 482 """test executing unicode strings"""
483 483 v = self.client[-1]
484 484 v.block=True
485 485 if sys.version_info[0] >= 3:
486 486 code="a='é'"
487 487 else:
488 488 code=u"a=u'é'"
489 489 v.execute(code)
490 490 self.assertEquals(v['a'], u'é')
491 491
492 492 def test_unicode_apply_result(self):
493 493 """test unicode apply results"""
494 494 v = self.client[-1]
495 495 r = v.apply_sync(lambda : u'é')
496 496 self.assertEquals(r, u'é')
497 497
498 498 def test_unicode_apply_arg(self):
499 499 """test passing unicode arguments to apply"""
500 500 v = self.client[-1]
501 501
502 502 @interactive
503 503 def check_unicode(a, check):
504 504 assert isinstance(a, unicode), "%r is not unicode"%a
505 505 assert isinstance(check, bytes), "%r is not bytes"%check
506 506 assert a.encode('utf8') == check, "%s != %s"%(a,check)
507 507
508 508 for s in [ u'é', u'ßø®∫',u'asdf' ]:
509 509 try:
510 510 v.apply_sync(check_unicode, s, s.encode('utf8'))
511 511 except error.RemoteError as e:
512 512 if e.ename == 'AssertionError':
513 513 self.fail(e.evalue)
514 514 else:
515 515 raise e
516 516
517 517 def test_map_reference(self):
518 518 """view.map(<Reference>, *seqs) should work"""
519 519 v = self.client[:]
520 520 v.scatter('n', self.client.ids, flatten=True)
521 521 v.execute("f = lambda x,y: x*y")
522 522 rf = pmod.Reference('f')
523 523 nlist = list(range(10))
524 524 mlist = nlist[::-1]
525 525 expected = [ m*n for m,n in zip(mlist, nlist) ]
526 526 result = v.map_sync(rf, mlist, nlist)
527 527 self.assertEquals(result, expected)
528 528
529 529 def test_apply_reference(self):
530 530 """view.apply(<Reference>, *args) should work"""
531 531 v = self.client[:]
532 532 v.scatter('n', self.client.ids, flatten=True)
533 533 v.execute("f = lambda x: n*x")
534 534 rf = pmod.Reference('f')
535 535 result = v.apply_sync(rf, 5)
536 536 expected = [ 5*id for id in self.client.ids ]
537 537 self.assertEquals(result, expected)
538 538
539 539 def test_eval_reference(self):
540 540 v = self.client[self.client.ids[0]]
541 541 v['g'] = range(5)
542 542 rg = pmod.Reference('g[0]')
543 543 echo = lambda x:x
544 544 self.assertEquals(v.apply_sync(echo, rg), 0)
545 545
546 546 def test_reference_nameerror(self):
547 547 v = self.client[self.client.ids[0]]
548 548 r = pmod.Reference('elvis_has_left')
549 549 echo = lambda x:x
550 550 self.assertRaisesRemote(NameError, v.apply_sync, echo, r)
551 551
552 552 def test_single_engine_map(self):
553 553 e0 = self.client[self.client.ids[0]]
554 554 r = range(5)
555 555 check = [ -1*i for i in r ]
556 556 result = e0.map_sync(lambda x: -1*x, r)
557 557 self.assertEquals(result, check)
558 558
559 559 def test_len(self):
560 560 """len(view) makes sense"""
561 561 e0 = self.client[self.client.ids[0]]
562 562 yield self.assertEquals(len(e0), 1)
563 563 v = self.client[:]
564 564 yield self.assertEquals(len(v), len(self.client.ids))
565 565 v = self.client.direct_view('all')
566 566 yield self.assertEquals(len(v), len(self.client.ids))
567 567 v = self.client[:2]
568 568 yield self.assertEquals(len(v), 2)
569 569 v = self.client[:1]
570 570 yield self.assertEquals(len(v), 1)
571 571 v = self.client.load_balanced_view()
572 572 yield self.assertEquals(len(v), len(self.client.ids))
573 573 # parametric tests seem to require manual closing?
574 574 self.client.close()
575 575
576 576
577 577 # begin execute tests
578 578 def _wait_for(self, f, timeout=10):
579 579 tic = time.time()
580 580 while time.time() <= tic + timeout:
581 581 if f():
582 582 return
583 583 time.sleep(0.1)
584 584 self.client.spin()
585 585 if not f():
586 586 print "Warning: Awaited condition never arrived"
587 587
588 588
589 589 def test_execute_reply(self):
590 590 e0 = self.client[self.client.ids[0]]
591 591 e0.block = True
592 592 ar = e0.execute("5", silent=False)
593 593 er = ar.get()
594 594 self._wait_for(lambda : bool(er.pyout))
595 595 self.assertEquals(str(er), "<ExecuteReply[%i]: 5>" % er.execution_count)
596 596 self.assertEquals(er.pyout['data']['text/plain'], '5')
597 597
598 598 def test_execute_reply_stdout(self):
599 599 e0 = self.client[self.client.ids[0]]
600 600 e0.block = True
601 601 ar = e0.execute("print (5)", silent=False)
602 602 er = ar.get()
603 603 self._wait_for(lambda : bool(er.stdout))
604 604 self.assertEquals(er.stdout.strip(), '5')
605 605
606 606 def test_execute_pyout(self):
607 607 """execute triggers pyout with silent=False"""
608 608 view = self.client[:]
609 609 ar = view.execute("5", silent=False, block=True)
610 610 self._wait_for(lambda : all(ar.pyout))
611 611
612 612 expected = [{'text/plain' : '5'}] * len(view)
613 613 mimes = [ out['data'] for out in ar.pyout ]
614 614 self.assertEquals(mimes, expected)
615 615
616 616 def test_execute_silent(self):
617 617 """execute does not trigger pyout with silent=True"""
618 618 view = self.client[:]
619 619 ar = view.execute("5", block=True)
620 620 expected = [None] * len(view)
621 621 self.assertEquals(ar.pyout, expected)
622 622
623 623 def test_execute_magic(self):
624 624 """execute accepts IPython commands"""
625 625 view = self.client[:]
626 626 view.execute("a = 5")
627 627 ar = view.execute("%whos", block=True)
628 628 # this will raise, if that failed
629 629 ar.get(5)
630 630 self._wait_for(lambda : all(ar.stdout))
631 631 for stdout in ar.stdout:
632 632 lines = stdout.splitlines()
633 633 self.assertEquals(lines[0].split(), ['Variable', 'Type', 'Data/Info'])
634 634 found = False
635 635 for line in lines[2:]:
636 636 split = line.split()
637 637 if split == ['a', 'int', '5']:
638 638 found = True
639 639 break
640 640 self.assertTrue(found, "whos output wrong: %s" % stdout)
641 641
642 642 def test_execute_displaypub(self):
643 643 """execute tracks display_pub output"""
644 644 view = self.client[:]
645 645 view.execute("from IPython.core.display import *")
646 646 ar = view.execute("[ display(i) for i in range(5) ]", block=True)
647 647
648 648 self._wait_for(lambda : all(len(er.outputs) >= 5 for er in ar))
649 649 expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
650 650 for outputs in ar.outputs:
651 651 mimes = [ out['data'] for out in outputs ]
652 652 self.assertEquals(mimes, expected)
653 653
654 654 def test_apply_displaypub(self):
655 655 """apply tracks display_pub output"""
656 656 view = self.client[:]
657 657 view.execute("from IPython.core.display import *")
658 658
659 659 @interactive
660 660 def publish():
661 661 [ display(i) for i in range(5) ]
662 662
663 663 ar = view.apply_async(publish)
664 664 ar.get(5)
665 665 self._wait_for(lambda : all(len(out) >= 5 for out in ar.outputs))
666 666 expected = [ {u'text/plain' : unicode(j)} for j in range(5) ]
667 667 for outputs in ar.outputs:
668 668 mimes = [ out['data'] for out in outputs ]
669 669 self.assertEquals(mimes, expected)
670 670
671 671 def test_execute_raises(self):
672 672 """exceptions in execute requests raise appropriately"""
673 673 view = self.client[-1]
674 674 ar = view.execute("1/0")
675 675 self.assertRaisesRemote(ZeroDivisionError, ar.get, 2)
676 676
677 677 @dec.skipif_not_matplotlib
678 678 def test_magic_pylab(self):
679 679 """%pylab works on engines"""
680 680 view = self.client[-1]
681 681 ar = view.execute("%pylab inline")
682 682 # at least check if this raised:
683 683 reply = ar.get(5)
684 684 # include imports, in case user config
685 685 ar = view.execute("plot(rand(100))", silent=False)
686 686 reply = ar.get(5)
687 687 self._wait_for(lambda : all(ar.outputs))
688 688 self.assertEquals(len(reply.outputs), 1)
689 689 output = reply.outputs[0]
690 690 self.assertTrue("data" in output)
691 691 data = output['data']
692 692 self.assertTrue("image/png" in data)
693 693
694 694
General Comments 0
You need to be logged in to leave comments. Login now