##// END OF EJS Templates
Add ExecutionMagics._run_with_profiler
Takafumi Arakaki -
Show More
@@ -1,1149 +1,1155 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Implementation of execution-related magic functions.
3 3 """
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (c) 2012 The IPython Development Team.
6 6 #
7 7 # Distributed under the terms of the Modified BSD License.
8 8 #
9 9 # The full license is in the file COPYING.txt, distributed with this software.
10 10 #-----------------------------------------------------------------------------
11 11
12 12 #-----------------------------------------------------------------------------
13 13 # Imports
14 14 #-----------------------------------------------------------------------------
15 15
16 16 # Stdlib
17 17 import __builtin__ as builtin_mod
18 18 import ast
19 19 import bdb
20 20 import os
21 21 import sys
22 22 import time
23 23 from StringIO import StringIO
24 24
25 25 # cProfile was added in Python2.5
26 26 try:
27 27 import cProfile as profile
28 28 import pstats
29 29 except ImportError:
30 30 # profile isn't bundled by default in Debian for license reasons
31 31 try:
32 32 import profile, pstats
33 33 except ImportError:
34 34 profile = pstats = None
35 35
36 36 # Our own packages
37 37 from IPython.core import debugger, oinspect
38 38 from IPython.core import magic_arguments
39 39 from IPython.core import page
40 40 from IPython.core.error import UsageError
41 41 from IPython.core.macro import Macro
42 42 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
43 43 line_cell_magic, on_off, needs_local_scope)
44 44 from IPython.testing.skipdoctest import skip_doctest
45 45 from IPython.utils import py3compat
46 46 from IPython.utils.contexts import preserve_keys
47 47 from IPython.utils.io import capture_output
48 48 from IPython.utils.ipstruct import Struct
49 49 from IPython.utils.module_paths import find_mod
50 50 from IPython.utils.path import get_py_filename, unquote_filename, shellglob
51 51 from IPython.utils.timing import clock, clock2
52 52 from IPython.utils.warn import warn, error
53 53
54 54
55 55 #-----------------------------------------------------------------------------
56 56 # Magic implementation classes
57 57 #-----------------------------------------------------------------------------
58 58
59 59 @magics_class
60 60 class ExecutionMagics(Magics):
61 61 """Magics related to code execution, debugging, profiling, etc.
62 62
63 63 """
64 64
65 65 def __init__(self, shell):
66 66 super(ExecutionMagics, self).__init__(shell)
67 67 if profile is None:
68 68 self.prun = self.profile_missing_notice
69 69 # Default execution function used to actually run user code.
70 70 self.default_runner = None
71 71
72 72 def profile_missing_notice(self, *args, **kwargs):
73 73 error("""\
74 74 The profile module could not be found. It has been removed from the standard
75 75 python packages because of its non-free license. To use profiling, install the
76 76 python-profiler package from non-free.""")
77 77
78 78 @skip_doctest
79 79 @line_cell_magic
80 def prun(self, parameter_s='', cell=None, user_mode=True,
81 opts=None,arg_lst=None,prog_ns=None, namespace=None):
80 def prun(self, parameter_s='', cell=None):
82 81
83 82 """Run a statement through the python code profiler.
84 83
85 84 Usage, in line mode:
86 85 %prun [options] statement
87 86
88 87 Usage, in cell mode:
89 88 %%prun [options] [statement]
90 89 code...
91 90 code...
92 91
93 92 In cell mode, the additional code lines are appended to the (possibly
94 93 empty) statement in the first line. Cell mode allows you to easily
95 94 profile multiline blocks without having to put them in a separate
96 95 function.
97 96
98 97 The given statement (which doesn't require quote marks) is run via the
99 98 python profiler in a manner similar to the profile.run() function.
100 99 Namespaces are internally managed to work correctly; profile.run
101 100 cannot be used in IPython because it makes certain assumptions about
102 101 namespaces which do not hold under IPython.
103 102
104 103 Options:
105 104
106 105 -l <limit>: you can place restrictions on what or how much of the
107 106 profile gets printed. The limit value can be:
108 107
109 108 * A string: only information for function names containing this string
110 109 is printed.
111 110
112 111 * An integer: only these many lines are printed.
113 112
114 113 * A float (between 0 and 1): this fraction of the report is printed
115 114 (for example, use a limit of 0.4 to see the topmost 40% only).
116 115
117 116 You can combine several limits with repeated use of the option. For
118 117 example, '-l __init__ -l 5' will print only the topmost 5 lines of
119 118 information about class constructors.
120 119
121 120 -r: return the pstats.Stats object generated by the profiling. This
122 121 object has all the information about the profile in it, and you can
123 122 later use it for further analysis or in other functions.
124 123
125 124 -s <key>: sort profile by given key. You can provide more than one key
126 125 by using the option several times: '-s key1 -s key2 -s key3...'. The
127 126 default sorting key is 'time'.
128 127
129 128 The following is copied verbatim from the profile documentation
130 129 referenced below:
131 130
132 131 When more than one key is provided, additional keys are used as
133 132 secondary criteria when the there is equality in all keys selected
134 133 before them.
135 134
136 135 Abbreviations can be used for any key names, as long as the
137 136 abbreviation is unambiguous. The following are the keys currently
138 137 defined:
139 138
140 139 Valid Arg Meaning
141 140 "calls" call count
142 141 "cumulative" cumulative time
143 142 "file" file name
144 143 "module" file name
145 144 "pcalls" primitive call count
146 145 "line" line number
147 146 "name" function name
148 147 "nfl" name/file/line
149 148 "stdname" standard name
150 149 "time" internal time
151 150
152 151 Note that all sorts on statistics are in descending order (placing
153 152 most time consuming items first), where as name, file, and line number
154 153 searches are in ascending order (i.e., alphabetical). The subtle
155 154 distinction between "nfl" and "stdname" is that the standard name is a
156 155 sort of the name as printed, which means that the embedded line
157 156 numbers get compared in an odd way. For example, lines 3, 20, and 40
158 157 would (if the file names were the same) appear in the string order
159 158 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
160 159 line numbers. In fact, sort_stats("nfl") is the same as
161 160 sort_stats("name", "file", "line").
162 161
163 162 -T <filename>: save profile results as shown on screen to a text
164 163 file. The profile is still shown on screen.
165 164
166 165 -D <filename>: save (via dump_stats) profile statistics to given
167 166 filename. This data is in a format understood by the pstats module, and
168 167 is generated by a call to the dump_stats() method of profile
169 168 objects. The profile is still shown on screen.
170 169
171 170 -q: suppress output to the pager. Best used with -T and/or -D above.
172 171
173 172 If you want to run complete programs under the profiler's control, use
174 173 '%run -p [prof_opts] filename.py [args to program]' where prof_opts
175 174 contains profiler specific options as described here.
176 175
177 176 You can read the complete documentation for the profile module with::
178 177
179 178 In [1]: import profile; profile.help()
180 179 """
180 return self._run_with_profiler(parameter_s, cell)
181
182 def _run_with_profiler(
183 self, parameter_s='', cell=None, user_mode=True,
184 opts=None, arg_lst=None, prog_ns=None, namespace=None):
181 185
182 186 opts_def = Struct(D=[''],l=[],s=['time'],T=[''])
183 187
184 188 if user_mode: # regular user call
185 189 opts,arg_str = self.parse_options(parameter_s,'D:l:rs:T:q',
186 190 list_all=True, posix=False)
187 191 namespace = self.shell.user_ns
188 192 if cell is not None:
189 193 arg_str += '\n' + cell
190 194 elif namespace is not None: # called to run a program by %run -p
191 195 arg_str = cell
192 196 else:
193 197 try:
194 198 filename = get_py_filename(arg_lst[0])
195 199 except IOError as e:
196 200 try:
197 201 msg = str(e)
198 202 except UnicodeError:
199 203 msg = e.message
200 204 error(msg)
201 205 return
202 206
203 207 arg_str = 'execfile(filename,prog_ns)'
204 208 namespace = {
205 209 'execfile': self.shell.safe_execfile,
206 210 'prog_ns': prog_ns,
207 211 'filename': filename
208 212 }
209 213
210 214 opts.merge(opts_def)
211 215
212 216 prof = profile.Profile()
213 217 try:
214 218 prof = prof.runctx(arg_str,namespace,namespace)
215 219 sys_exit = ''
216 220 except SystemExit:
217 221 sys_exit = """*** SystemExit exception caught in code being profiled."""
218 222
219 223 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
220 224
221 225 lims = opts.l
222 226 if lims:
223 227 lims = [] # rebuild lims with ints/floats/strings
224 228 for lim in opts.l:
225 229 try:
226 230 lims.append(int(lim))
227 231 except ValueError:
228 232 try:
229 233 lims.append(float(lim))
230 234 except ValueError:
231 235 lims.append(lim)
232 236
233 237 # Trap output.
234 238 stdout_trap = StringIO()
235 239 stats_stream = stats.stream
236 240 try:
237 241 stats.stream = stdout_trap
238 242 stats.print_stats(*lims)
239 243 finally:
240 244 stats.stream = stats_stream
241 245
242 246 output = stdout_trap.getvalue()
243 247 output = output.rstrip()
244 248
245 249 if 'q' not in opts:
246 250 page.page(output)
247 251 print sys_exit,
248 252
249 253 dump_file = opts.D[0]
250 254 text_file = opts.T[0]
251 255 if dump_file:
252 256 dump_file = unquote_filename(dump_file)
253 257 prof.dump_stats(dump_file)
254 258 print '\n*** Profile stats marshalled to file',\
255 259 repr(dump_file)+'.',sys_exit
256 260 if text_file:
257 261 text_file = unquote_filename(text_file)
258 262 pfile = open(text_file,'w')
259 263 pfile.write(output)
260 264 pfile.close()
261 265 print '\n*** Profile printout saved to text file',\
262 266 repr(text_file)+'.',sys_exit
263 267
264 268 if 'r' in opts:
265 269 return stats
266 270 else:
267 271 return None
268 272
269 273 @line_magic
270 274 def pdb(self, parameter_s=''):
271 275 """Control the automatic calling of the pdb interactive debugger.
272 276
273 277 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
274 278 argument it works as a toggle.
275 279
276 280 When an exception is triggered, IPython can optionally call the
277 281 interactive pdb debugger after the traceback printout. %pdb toggles
278 282 this feature on and off.
279 283
280 284 The initial state of this feature is set in your configuration
281 285 file (the option is ``InteractiveShell.pdb``).
282 286
283 287 If you want to just activate the debugger AFTER an exception has fired,
284 288 without having to type '%pdb on' and rerunning your code, you can use
285 289 the %debug magic."""
286 290
287 291 par = parameter_s.strip().lower()
288 292
289 293 if par:
290 294 try:
291 295 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
292 296 except KeyError:
293 297 print ('Incorrect argument. Use on/1, off/0, '
294 298 'or nothing for a toggle.')
295 299 return
296 300 else:
297 301 # toggle
298 302 new_pdb = not self.shell.call_pdb
299 303
300 304 # set on the shell
301 305 self.shell.call_pdb = new_pdb
302 306 print 'Automatic pdb calling has been turned',on_off(new_pdb)
303 307
304 308 @line_magic
305 309 def debug(self, parameter_s=''):
306 310 """Activate the interactive debugger in post-mortem mode.
307 311
308 312 If an exception has just occurred, this lets you inspect its stack
309 313 frames interactively. Note that this will always work only on the last
310 314 traceback that occurred, so you must call this quickly after an
311 315 exception that you wish to inspect has fired, because if another one
312 316 occurs, it clobbers the previous one.
313 317
314 318 If you want IPython to automatically do this on every exception, see
315 319 the %pdb magic for more details.
316 320 """
317 321 self.shell.debugger(force=True)
318 322
319 323 @line_magic
320 324 def tb(self, s):
321 325 """Print the last traceback with the currently active exception mode.
322 326
323 327 See %xmode for changing exception reporting modes."""
324 328 self.shell.showtraceback()
325 329
326 330 @skip_doctest
327 331 @line_magic
328 332 def run(self, parameter_s='', runner=None,
329 333 file_finder=get_py_filename):
330 334 """Run the named file inside IPython as a program.
331 335
332 336 Usage:\\
333 337 %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options] -G] file [args]
334 338
335 339 Parameters after the filename are passed as command-line arguments to
336 340 the program (put in sys.argv). Then, control returns to IPython's
337 341 prompt.
338 342
339 343 This is similar to running at a system prompt:\\
340 344 $ python file args\\
341 345 but with the advantage of giving you IPython's tracebacks, and of
342 346 loading all variables into your interactive namespace for further use
343 347 (unless -p is used, see below).
344 348
345 349 The file is executed in a namespace initially consisting only of
346 350 __name__=='__main__' and sys.argv constructed as indicated. It thus
347 351 sees its environment as if it were being run as a stand-alone program
348 352 (except for sharing global objects such as previously imported
349 353 modules). But after execution, the IPython interactive namespace gets
350 354 updated with all variables defined in the program (except for __name__
351 355 and sys.argv). This allows for very convenient loading of code for
352 356 interactive work, while giving each program a 'clean sheet' to run in.
353 357
354 358 Arguments are expanded using shell-like glob match. Patterns
355 359 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
356 360 tilde '~' will be expanded into user's home directory. Unlike
357 361 real shells, quotation does not suppress expansions. Use
358 362 *two* back slashes (e.g., '\\\\*') to suppress expansions.
359 363 To completely disable these expansions, you can use -G flag.
360 364
361 365 Options:
362 366
363 367 -n: __name__ is NOT set to '__main__', but to the running file's name
364 368 without extension (as python does under import). This allows running
365 369 scripts and reloading the definitions in them without calling code
366 370 protected by an ' if __name__ == "__main__" ' clause.
367 371
368 372 -i: run the file in IPython's namespace instead of an empty one. This
369 373 is useful if you are experimenting with code written in a text editor
370 374 which depends on variables defined interactively.
371 375
372 376 -e: ignore sys.exit() calls or SystemExit exceptions in the script
373 377 being run. This is particularly useful if IPython is being used to
374 378 run unittests, which always exit with a sys.exit() call. In such
375 379 cases you are interested in the output of the test results, not in
376 380 seeing a traceback of the unittest module.
377 381
378 382 -t: print timing information at the end of the run. IPython will give
379 383 you an estimated CPU time consumption for your script, which under
380 384 Unix uses the resource module to avoid the wraparound problems of
381 385 time.clock(). Under Unix, an estimate of time spent on system tasks
382 386 is also given (for Windows platforms this is reported as 0.0).
383 387
384 388 If -t is given, an additional -N<N> option can be given, where <N>
385 389 must be an integer indicating how many times you want the script to
386 390 run. The final timing report will include total and per run results.
387 391
388 392 For example (testing the script uniq_stable.py)::
389 393
390 394 In [1]: run -t uniq_stable
391 395
392 396 IPython CPU timings (estimated):\\
393 397 User : 0.19597 s.\\
394 398 System: 0.0 s.\\
395 399
396 400 In [2]: run -t -N5 uniq_stable
397 401
398 402 IPython CPU timings (estimated):\\
399 403 Total runs performed: 5\\
400 404 Times : Total Per run\\
401 405 User : 0.910862 s, 0.1821724 s.\\
402 406 System: 0.0 s, 0.0 s.
403 407
404 408 -d: run your program under the control of pdb, the Python debugger.
405 409 This allows you to execute your program step by step, watch variables,
406 410 etc. Internally, what IPython does is similar to calling:
407 411
408 412 pdb.run('execfile("YOURFILENAME")')
409 413
410 414 with a breakpoint set on line 1 of your file. You can change the line
411 415 number for this automatic breakpoint to be <N> by using the -bN option
412 416 (where N must be an integer). For example::
413 417
414 418 %run -d -b40 myscript
415 419
416 420 will set the first breakpoint at line 40 in myscript.py. Note that
417 421 the first breakpoint must be set on a line which actually does
418 422 something (not a comment or docstring) for it to stop execution.
419 423
420 424 Or you can specify a breakpoint in a different file::
421 425
422 426 %run -d -b myotherfile.py:20 myscript
423 427
424 428 When the pdb debugger starts, you will see a (Pdb) prompt. You must
425 429 first enter 'c' (without quotes) to start execution up to the first
426 430 breakpoint.
427 431
428 432 Entering 'help' gives information about the use of the debugger. You
429 433 can easily see pdb's full documentation with "import pdb;pdb.help()"
430 434 at a prompt.
431 435
432 436 -p: run program under the control of the Python profiler module (which
433 437 prints a detailed report of execution times, function calls, etc).
434 438
435 439 You can pass other options after -p which affect the behavior of the
436 440 profiler itself. See the docs for %prun for details.
437 441
438 442 In this mode, the program's variables do NOT propagate back to the
439 443 IPython interactive namespace (because they remain in the namespace
440 444 where the profiler executes them).
441 445
442 446 Internally this triggers a call to %prun, see its documentation for
443 447 details on the options available specifically for profiling.
444 448
445 449 There is one special usage for which the text above doesn't apply:
446 450 if the filename ends with .ipy, the file is run as ipython script,
447 451 just as if the commands were written on IPython prompt.
448 452
449 453 -m: specify module name to load instead of script path. Similar to
450 454 the -m option for the python interpreter. Use this option last if you
451 455 want to combine with other %run options. Unlike the python interpreter
452 456 only source modules are allowed no .pyc or .pyo files.
453 457 For example::
454 458
455 459 %run -m example
456 460
457 461 will run the example module.
458 462
459 463 -G: disable shell-like glob expansion of arguments.
460 464
461 465 """
462 466
463 467 # get arguments and set sys.argv for program to be run.
464 468 opts, arg_lst = self.parse_options(parameter_s,
465 469 'nidtN:b:pD:l:rs:T:em:G',
466 470 mode='list', list_all=1)
467 471 if "m" in opts:
468 472 modulename = opts["m"][0]
469 473 modpath = find_mod(modulename)
470 474 if modpath is None:
471 475 warn('%r is not a valid modulename on sys.path'%modulename)
472 476 return
473 477 arg_lst = [modpath] + arg_lst
474 478 try:
475 479 filename = file_finder(arg_lst[0])
476 480 except IndexError:
477 481 warn('you must provide at least a filename.')
478 482 print '\n%run:\n', oinspect.getdoc(self.run)
479 483 return
480 484 except IOError as e:
481 485 try:
482 486 msg = str(e)
483 487 except UnicodeError:
484 488 msg = e.message
485 489 error(msg)
486 490 return
487 491
488 492 if filename.lower().endswith('.ipy'):
489 493 with preserve_keys(self.shell.user_ns, '__file__'):
490 494 self.shell.user_ns['__file__'] = filename
491 495 self.shell.safe_execfile_ipy(filename)
492 496 return
493 497
494 498 # Control the response to exit() calls made by the script being run
495 499 exit_ignore = 'e' in opts
496 500
497 501 # Make sure that the running script gets a proper sys.argv as if it
498 502 # were run from a system shell.
499 503 save_argv = sys.argv # save it for later restoring
500 504
501 505 if 'G' in opts:
502 506 args = arg_lst[1:]
503 507 else:
504 508 # tilde and glob expansion
505 509 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
506 510
507 511 sys.argv = [filename] + args # put in the proper filename
508 512 # protect sys.argv from potential unicode strings on Python 2:
509 513 if not py3compat.PY3:
510 514 sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
511 515
512 516 if 'i' in opts:
513 517 # Run in user's interactive namespace
514 518 prog_ns = self.shell.user_ns
515 519 __name__save = self.shell.user_ns['__name__']
516 520 prog_ns['__name__'] = '__main__'
517 521 main_mod = self.shell.new_main_mod(prog_ns)
518 522 else:
519 523 # Run in a fresh, empty namespace
520 524 if 'n' in opts:
521 525 name = os.path.splitext(os.path.basename(filename))[0]
522 526 else:
523 527 name = '__main__'
524 528
525 529 main_mod = self.shell.new_main_mod()
526 530 prog_ns = main_mod.__dict__
527 531 prog_ns['__name__'] = name
528 532
529 533 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
530 534 # set the __file__ global in the script's namespace
531 535 prog_ns['__file__'] = filename
532 536
533 537 # pickle fix. See interactiveshell for an explanation. But we need to
534 538 # make sure that, if we overwrite __main__, we replace it at the end
535 539 main_mod_name = prog_ns['__name__']
536 540
537 541 if main_mod_name == '__main__':
538 542 restore_main = sys.modules['__main__']
539 543 else:
540 544 restore_main = False
541 545
542 546 # This needs to be undone at the end to prevent holding references to
543 547 # every single object ever created.
544 548 sys.modules[main_mod_name] = main_mod
545 549
546 550 if 'p' in opts or 'd' in opts:
547 551 if 'm' in opts:
548 552 code = 'run_module(modulename, prog_ns)'
549 553 code_ns = {
550 554 'run_module': self.shell.safe_run_module,
551 555 'prog_ns': prog_ns,
552 556 'modulename': modulename,
553 557 }
554 558 else:
555 559 code = 'execfile(filename, prog_ns)'
556 560 code_ns = {
557 561 'execfile': self.shell.safe_execfile,
558 562 'prog_ns': prog_ns,
559 563 'filename': get_py_filename(filename),
560 564 }
561 565
562 566 try:
563 567 stats = None
564 568 with self.shell.readline_no_record:
565 569 if 'p' in opts:
566 stats = self.prun('', code, False, opts, namespace=code_ns)
570 stats = self._run_with_profiler(
571 cell=code, user_mode=False, opts=opts,
572 namespace=code_ns)
567 573 else:
568 574 if 'd' in opts:
569 575 self._run_with_debugger(
570 576 code, code_ns, opts.get('b', ['1'])[0], filename)
571 577 else:
572 578 if 'm' in opts:
573 579 def run():
574 580 self.shell.safe_run_module(modulename, prog_ns)
575 581 else:
576 582 if runner is None:
577 583 runner = self.default_runner
578 584 if runner is None:
579 585 runner = self.shell.safe_execfile
580 586
581 587 def run():
582 588 runner(filename, prog_ns, prog_ns,
583 589 exit_ignore=exit_ignore)
584 590
585 591 if 't' in opts:
586 592 # timed execution
587 593 try:
588 594 nruns = int(opts['N'][0])
589 595 if nruns < 1:
590 596 error('Number of runs must be >=1')
591 597 return
592 598 except (KeyError):
593 599 nruns = 1
594 600 self._run_with_timing(run, nruns)
595 601 else:
596 602 # regular execution
597 603 run()
598 604
599 605 if 'i' in opts:
600 606 self.shell.user_ns['__name__'] = __name__save
601 607 else:
602 608 # The shell MUST hold a reference to prog_ns so after %run
603 609 # exits, the python deletion mechanism doesn't zero it out
604 610 # (leaving dangling references).
605 611 self.shell.cache_main_mod(prog_ns, filename)
606 612 # update IPython interactive namespace
607 613
608 614 # Some forms of read errors on the file may mean the
609 615 # __name__ key was never set; using pop we don't have to
610 616 # worry about a possible KeyError.
611 617 prog_ns.pop('__name__', None)
612 618
613 619 with preserve_keys(self.shell.user_ns, '__file__'):
614 620 self.shell.user_ns.update(prog_ns)
615 621 finally:
616 622 # It's a bit of a mystery why, but __builtins__ can change from
617 623 # being a module to becoming a dict missing some key data after
618 624 # %run. As best I can see, this is NOT something IPython is doing
619 625 # at all, and similar problems have been reported before:
620 626 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
621 627 # Since this seems to be done by the interpreter itself, the best
622 628 # we can do is to at least restore __builtins__ for the user on
623 629 # exit.
624 630 self.shell.user_ns['__builtins__'] = builtin_mod
625 631
626 632 # Ensure key global structures are restored
627 633 sys.argv = save_argv
628 634 if restore_main:
629 635 sys.modules['__main__'] = restore_main
630 636 else:
631 637 # Remove from sys.modules the reference to main_mod we'd
632 638 # added. Otherwise it will trap references to objects
633 639 # contained therein.
634 640 del sys.modules[main_mod_name]
635 641
636 642 return stats
637 643
638 644 def _run_with_debugger(self, code, code_ns, break_point, filename):
639 645 deb = debugger.Pdb(self.shell.colors)
640 646 # reset Breakpoint state, which is moronically kept
641 647 # in a class
642 648 bdb.Breakpoint.next = 1
643 649 bdb.Breakpoint.bplist = {}
644 650 bdb.Breakpoint.bpbynumber = [None]
645 651 # Set an initial breakpoint to stop execution
646 652 maxtries = 10
647 653 bp_file, bp_line = parse_breakpoint(break_point, filename)
648 654 checkline = deb.checkline(bp_file, bp_line)
649 655 if not checkline:
650 656 for bp in range(bp_line + 1, bp_line + maxtries + 1):
651 657 if deb.checkline(bp_file, bp):
652 658 break
653 659 else:
654 660 msg = ("\nI failed to find a valid line to set "
655 661 "a breakpoint\n"
656 662 "after trying up to line: %s.\n"
657 663 "Please set a valid breakpoint manually "
658 664 "with the -b option." % bp)
659 665 error(msg)
660 666 return
661 667 # if we find a good linenumber, set the breakpoint
662 668 deb.do_break('%s:%s' % (bp_file, bp_line))
663 669
664 670 # Mimic Pdb._runscript(...)
665 671 deb._wait_for_mainpyfile = True
666 672 deb.mainpyfile = deb.canonic(filename)
667 673
668 674 # Start file run
669 675 print "NOTE: Enter 'c' at the",
670 676 print "%s prompt to start your script." % deb.prompt
671 677 try:
672 678 #save filename so it can be used by methods on the deb object
673 679 deb._exec_filename = filename
674 680 deb.run(code, code_ns)
675 681
676 682 except:
677 683 etype, value, tb = sys.exc_info()
678 684 # Skip three frames in the traceback: the %run one,
679 685 # one inside bdb.py, and the command-line typed by the
680 686 # user (run by exec in pdb itself).
681 687 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
682 688
683 689 @staticmethod
684 690 def _run_with_timing(run, nruns):
685 691 twall0 = time.time()
686 692 if nruns == 1:
687 693 t0 = clock2()
688 694 run()
689 695 t1 = clock2()
690 696 t_usr = t1[0] - t0[0]
691 697 t_sys = t1[1] - t0[1]
692 698 print "\nIPython CPU timings (estimated):"
693 699 print " User : %10.2f s." % t_usr
694 700 print " System : %10.2f s." % t_sys
695 701 else:
696 702 runs = range(nruns)
697 703 t0 = clock2()
698 704 for nr in runs:
699 705 run()
700 706 t1 = clock2()
701 707 t_usr = t1[0] - t0[0]
702 708 t_sys = t1[1] - t0[1]
703 709 print "\nIPython CPU timings (estimated):"
704 710 print "Total runs performed:", nruns
705 711 print " Times : %10s %10s" % ('Total', 'Per run')
706 712 print " User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns)
707 713 print " System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns)
708 714 twall1 = time.time()
709 715 print "Wall time: %10.2f s." % (twall1 - twall0)
710 716
711 717 @skip_doctest
712 718 @line_cell_magic
713 719 def timeit(self, line='', cell=None):
714 720 """Time execution of a Python statement or expression
715 721
716 722 Usage, in line mode:
717 723 %timeit [-n<N> -r<R> [-t|-c]] statement
718 724 or in cell mode:
719 725 %%timeit [-n<N> -r<R> [-t|-c]] setup_code
720 726 code
721 727 code...
722 728
723 729 Time execution of a Python statement or expression using the timeit
724 730 module. This function can be used both as a line and cell magic:
725 731
726 732 - In line mode you can time a single-line statement (though multiple
727 733 ones can be chained with using semicolons).
728 734
729 735 - In cell mode, the statement in the first line is used as setup code
730 736 (executed but not timed) and the body of the cell is timed. The cell
731 737 body has access to any variables created in the setup code.
732 738
733 739 Options:
734 740 -n<N>: execute the given statement <N> times in a loop. If this value
735 741 is not given, a fitting value is chosen.
736 742
737 743 -r<R>: repeat the loop iteration <R> times and take the best result.
738 744 Default: 3
739 745
740 746 -t: use time.time to measure the time, which is the default on Unix.
741 747 This function measures wall time.
742 748
743 749 -c: use time.clock to measure the time, which is the default on
744 750 Windows and measures wall time. On Unix, resource.getrusage is used
745 751 instead and returns the CPU user time.
746 752
747 753 -p<P>: use a precision of <P> digits to display the timing result.
748 754 Default: 3
749 755
750 756
751 757 Examples
752 758 --------
753 759 ::
754 760
755 761 In [1]: %timeit pass
756 762 10000000 loops, best of 3: 53.3 ns per loop
757 763
758 764 In [2]: u = None
759 765
760 766 In [3]: %timeit u is None
761 767 10000000 loops, best of 3: 184 ns per loop
762 768
763 769 In [4]: %timeit -r 4 u == None
764 770 1000000 loops, best of 4: 242 ns per loop
765 771
766 772 In [5]: import time
767 773
768 774 In [6]: %timeit -n1 time.sleep(2)
769 775 1 loops, best of 3: 2 s per loop
770 776
771 777
772 778 The times reported by %timeit will be slightly higher than those
773 779 reported by the timeit.py script when variables are accessed. This is
774 780 due to the fact that %timeit executes the statement in the namespace
775 781 of the shell, compared with timeit.py, which uses a single setup
776 782 statement to import function or create variables. Generally, the bias
777 783 does not matter as long as results from timeit.py are not mixed with
778 784 those from %timeit."""
779 785
780 786 import timeit
781 787
782 788 opts, stmt = self.parse_options(line,'n:r:tcp:',
783 789 posix=False, strict=False)
784 790 if stmt == "" and cell is None:
785 791 return
786 792
787 793 timefunc = timeit.default_timer
788 794 number = int(getattr(opts, "n", 0))
789 795 repeat = int(getattr(opts, "r", timeit.default_repeat))
790 796 precision = int(getattr(opts, "p", 3))
791 797 if hasattr(opts, "t"):
792 798 timefunc = time.time
793 799 if hasattr(opts, "c"):
794 800 timefunc = clock
795 801
796 802 timer = timeit.Timer(timer=timefunc)
797 803 # this code has tight coupling to the inner workings of timeit.Timer,
798 804 # but is there a better way to achieve that the code stmt has access
799 805 # to the shell namespace?
800 806 transform = self.shell.input_splitter.transform_cell
801 807
802 808 if cell is None:
803 809 # called as line magic
804 810 ast_setup = ast.parse("pass")
805 811 ast_stmt = ast.parse(transform(stmt))
806 812 else:
807 813 ast_setup = ast.parse(transform(stmt))
808 814 ast_stmt = ast.parse(transform(cell))
809 815
810 816 ast_setup = self.shell.transform_ast(ast_setup)
811 817 ast_stmt = self.shell.transform_ast(ast_stmt)
812 818
813 819 # This codestring is taken from timeit.template - we fill it in as an
814 820 # AST, so that we can apply our AST transformations to the user code
815 821 # without affecting the timing code.
816 822 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
817 823 ' setup\n'
818 824 ' _t0 = _timer()\n'
819 825 ' for _i in _it:\n'
820 826 ' stmt\n'
821 827 ' _t1 = _timer()\n'
822 828 ' return _t1 - _t0\n')
823 829
824 830 class TimeitTemplateFiller(ast.NodeTransformer):
825 831 "This is quite tightly tied to the template definition above."
826 832 def visit_FunctionDef(self, node):
827 833 "Fill in the setup statement"
828 834 self.generic_visit(node)
829 835 if node.name == "inner":
830 836 node.body[:1] = ast_setup.body
831 837
832 838 return node
833 839
834 840 def visit_For(self, node):
835 841 "Fill in the statement to be timed"
836 842 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
837 843 node.body = ast_stmt.body
838 844 return node
839 845
840 846 timeit_ast = TimeitTemplateFiller().visit(timeit_ast_template)
841 847 timeit_ast = ast.fix_missing_locations(timeit_ast)
842 848
843 849 # Track compilation time so it can be reported if too long
844 850 # Minimum time above which compilation time will be reported
845 851 tc_min = 0.1
846 852
847 853 t0 = clock()
848 854 code = compile(timeit_ast, "<magic-timeit>", "exec")
849 855 tc = clock()-t0
850 856
851 857 ns = {}
852 858 exec code in self.shell.user_ns, ns
853 859 timer.inner = ns["inner"]
854 860
855 861 if number == 0:
856 862 # determine number so that 0.2 <= total time < 2.0
857 863 number = 1
858 864 for i in range(1, 10):
859 865 if timer.timeit(number) >= 0.2:
860 866 break
861 867 number *= 10
862 868
863 869 best = min(timer.repeat(repeat, number)) / number
864 870
865 871 print u"%d loops, best of %d: %s per loop" % (number, repeat,
866 872 _format_time(best, precision))
867 873 if tc > tc_min:
868 874 print "Compiler time: %.2f s" % tc
869 875
870 876 @skip_doctest
871 877 @needs_local_scope
872 878 @line_cell_magic
873 879 def time(self,line='', cell=None, local_ns=None):
874 880 """Time execution of a Python statement or expression.
875 881
876 882 The CPU and wall clock times are printed, and the value of the
877 883 expression (if any) is returned. Note that under Win32, system time
878 884 is always reported as 0, since it can not be measured.
879 885
880 886 This function can be used both as a line and cell magic:
881 887
882 888 - In line mode you can time a single-line statement (though multiple
883 889 ones can be chained with using semicolons).
884 890
885 891 - In cell mode, you can time the cell body (a directly
886 892 following statement raises an error).
887 893
888 894 This function provides very basic timing functionality. Use the timeit
889 895 magic for more controll over the measurement.
890 896
891 897 Examples
892 898 --------
893 899 ::
894 900
895 901 In [1]: %time 2**128
896 902 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
897 903 Wall time: 0.00
898 904 Out[1]: 340282366920938463463374607431768211456L
899 905
900 906 In [2]: n = 1000000
901 907
902 908 In [3]: %time sum(range(n))
903 909 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
904 910 Wall time: 1.37
905 911 Out[3]: 499999500000L
906 912
907 913 In [4]: %time print 'hello world'
908 914 hello world
909 915 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
910 916 Wall time: 0.00
911 917
912 918 Note that the time needed by Python to compile the given expression
913 919 will be reported if it is more than 0.1s. In this example, the
914 920 actual exponentiation is done by Python at compilation time, so while
915 921 the expression can take a noticeable amount of time to compute, that
916 922 time is purely due to the compilation:
917 923
918 924 In [5]: %time 3**9999;
919 925 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
920 926 Wall time: 0.00 s
921 927
922 928 In [6]: %time 3**999999;
923 929 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
924 930 Wall time: 0.00 s
925 931 Compiler : 0.78 s
926 932 """
927 933
928 934 # fail immediately if the given expression can't be compiled
929 935
930 936 if line and cell:
931 937 raise UsageError("Can't use statement directly after '%%time'!")
932 938
933 939 if cell:
934 940 expr = self.shell.prefilter(cell,False)
935 941 else:
936 942 expr = self.shell.prefilter(line,False)
937 943
938 944 # Minimum time above which parse time will be reported
939 945 tp_min = 0.1
940 946
941 947 t0 = clock()
942 948 expr_ast = ast.parse(expr)
943 949 tp = clock()-t0
944 950
945 951 # Apply AST transformations
946 952 expr_ast = self.shell.transform_ast(expr_ast)
947 953
948 954 # Minimum time above which compilation time will be reported
949 955 tc_min = 0.1
950 956
951 957 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
952 958 mode = 'eval'
953 959 source = '<timed eval>'
954 960 expr_ast = ast.Expression(expr_ast.body[0].value)
955 961 else:
956 962 mode = 'exec'
957 963 source = '<timed exec>'
958 964 t0 = clock()
959 965 code = compile(expr_ast, source, mode)
960 966 tc = clock()-t0
961 967
962 968 # skew measurement as little as possible
963 969 glob = self.shell.user_ns
964 970 wtime = time.time
965 971 # time execution
966 972 wall_st = wtime()
967 973 if mode=='eval':
968 974 st = clock2()
969 975 out = eval(code, glob, local_ns)
970 976 end = clock2()
971 977 else:
972 978 st = clock2()
973 979 exec code in glob, local_ns
974 980 end = clock2()
975 981 out = None
976 982 wall_end = wtime()
977 983 # Compute actual times and report
978 984 wall_time = wall_end-wall_st
979 985 cpu_user = end[0]-st[0]
980 986 cpu_sys = end[1]-st[1]
981 987 cpu_tot = cpu_user+cpu_sys
982 988 # On windows cpu_sys is always zero, so no new information to the next print
983 989 if sys.platform != 'win32':
984 990 print "CPU times: user %s, sys: %s, total: %s" % \
985 991 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot))
986 992 print "Wall time: %s" % _format_time(wall_time)
987 993 if tc > tc_min:
988 994 print "Compiler : %s" % _format_time(tc)
989 995 if tp > tp_min:
990 996 print "Parser : %s" % _format_time(tp)
991 997 return out
992 998
993 999 @skip_doctest
994 1000 @line_magic
995 1001 def macro(self, parameter_s=''):
996 1002 """Define a macro for future re-execution. It accepts ranges of history,
997 1003 filenames or string objects.
998 1004
999 1005 Usage:\\
1000 1006 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1001 1007
1002 1008 Options:
1003 1009
1004 1010 -r: use 'raw' input. By default, the 'processed' history is used,
1005 1011 so that magics are loaded in their transformed version to valid
1006 1012 Python. If this option is given, the raw input as typed as the
1007 1013 command line is used instead.
1008 1014
1009 1015 This will define a global variable called `name` which is a string
1010 1016 made of joining the slices and lines you specify (n1,n2,... numbers
1011 1017 above) from your input history into a single string. This variable
1012 1018 acts like an automatic function which re-executes those lines as if
1013 1019 you had typed them. You just type 'name' at the prompt and the code
1014 1020 executes.
1015 1021
1016 1022 The syntax for indicating input ranges is described in %history.
1017 1023
1018 1024 Note: as a 'hidden' feature, you can also use traditional python slice
1019 1025 notation, where N:M means numbers N through M-1.
1020 1026
1021 1027 For example, if your history contains (%hist prints it)::
1022 1028
1023 1029 44: x=1
1024 1030 45: y=3
1025 1031 46: z=x+y
1026 1032 47: print x
1027 1033 48: a=5
1028 1034 49: print 'x',x,'y',y
1029 1035
1030 1036 you can create a macro with lines 44 through 47 (included) and line 49
1031 1037 called my_macro with::
1032 1038
1033 1039 In [55]: %macro my_macro 44-47 49
1034 1040
1035 1041 Now, typing `my_macro` (without quotes) will re-execute all this code
1036 1042 in one pass.
1037 1043
1038 1044 You don't need to give the line-numbers in order, and any given line
1039 1045 number can appear multiple times. You can assemble macros with any
1040 1046 lines from your input history in any order.
1041 1047
1042 1048 The macro is a simple object which holds its value in an attribute,
1043 1049 but IPython's display system checks for macros and executes them as
1044 1050 code instead of printing them when you type their name.
1045 1051
1046 1052 You can view a macro's contents by explicitly printing it with::
1047 1053
1048 1054 print macro_name
1049 1055
1050 1056 """
1051 1057 opts,args = self.parse_options(parameter_s,'r',mode='list')
1052 1058 if not args: # List existing macros
1053 1059 return sorted(k for k,v in self.shell.user_ns.iteritems() if\
1054 1060 isinstance(v, Macro))
1055 1061 if len(args) == 1:
1056 1062 raise UsageError(
1057 1063 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1058 1064 name, codefrom = args[0], " ".join(args[1:])
1059 1065
1060 1066 #print 'rng',ranges # dbg
1061 1067 try:
1062 1068 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1063 1069 except (ValueError, TypeError) as e:
1064 1070 print e.args[0]
1065 1071 return
1066 1072 macro = Macro(lines)
1067 1073 self.shell.define_macro(name, macro)
1068 1074 print 'Macro `%s` created. To execute, type its name (without quotes).' % name
1069 1075 print '=== Macro contents: ==='
1070 1076 print macro,
1071 1077
1072 1078 @magic_arguments.magic_arguments()
1073 1079 @magic_arguments.argument('output', type=str, default='', nargs='?',
1074 1080 help="""The name of the variable in which to store output.
1075 1081 This is a utils.io.CapturedIO object with stdout/err attributes
1076 1082 for the text of the captured output.
1077 1083
1078 1084 CapturedOutput also has a show() method for displaying the output,
1079 1085 and __call__ as well, so you can use that to quickly display the
1080 1086 output.
1081 1087
1082 1088 If unspecified, captured output is discarded.
1083 1089 """
1084 1090 )
1085 1091 @magic_arguments.argument('--no-stderr', action="store_true",
1086 1092 help="""Don't capture stderr."""
1087 1093 )
1088 1094 @magic_arguments.argument('--no-stdout', action="store_true",
1089 1095 help="""Don't capture stdout."""
1090 1096 )
1091 1097 @cell_magic
1092 1098 def capture(self, line, cell):
1093 1099 """run the cell, capturing stdout/err"""
1094 1100 args = magic_arguments.parse_argstring(self.capture, line)
1095 1101 out = not args.no_stdout
1096 1102 err = not args.no_stderr
1097 1103 with capture_output(out, err) as io:
1098 1104 self.shell.run_cell(cell)
1099 1105 if args.output:
1100 1106 self.shell.user_ns[args.output] = io
1101 1107
1102 1108 def parse_breakpoint(text, current_file):
1103 1109 '''Returns (file, line) for file:line and (current_file, line) for line'''
1104 1110 colon = text.find(':')
1105 1111 if colon == -1:
1106 1112 return current_file, int(text)
1107 1113 else:
1108 1114 return text[:colon], int(text[colon+1:])
1109 1115
1110 1116 def _format_time(timespan, precision=3):
1111 1117 """Formats the timespan in a human readable form"""
1112 1118 import math
1113 1119
1114 1120 if timespan >= 60.0:
1115 1121 # we have more than a minute, format that in a human readable form
1116 1122 # Idea from http://snipplr.com/view/5713/
1117 1123 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1118 1124 time = []
1119 1125 leftover = timespan
1120 1126 for suffix, length in parts:
1121 1127 value = int(leftover / length)
1122 1128 if value > 0:
1123 1129 leftover = leftover % length
1124 1130 time.append(u'%s%s' % (str(value), suffix))
1125 1131 if leftover < 1:
1126 1132 break
1127 1133 return " ".join(time)
1128 1134
1129 1135
1130 1136 # Unfortunately the unicode 'micro' symbol can cause problems in
1131 1137 # certain terminals.
1132 1138 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1133 1139 # Try to prevent crashes by being more secure than it needs to
1134 1140 # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
1135 1141 units = [u"s", u"ms",u'us',"ns"] # the save value
1136 1142 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1137 1143 try:
1138 1144 u'\xb5'.encode(sys.stdout.encoding)
1139 1145 units = [u"s", u"ms",u'\xb5s',"ns"]
1140 1146 except:
1141 1147 pass
1142 1148 scaling = [1, 1e3, 1e6, 1e9]
1143 1149
1144 1150 if timespan > 0.0:
1145 1151 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1146 1152 else:
1147 1153 order = 3
1148 1154 ret = u"%.*g %s" % (precision, timespan * scaling[order], units[order])
1149 1155 return ret
General Comments 0
You need to be logged in to leave comments. Login now