##// END OF EJS Templates
remove sys_version for Python 3...
Paul Ivanov -
Show More
@@ -1,631 +1,630 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Pdb debugger class.
4 4
5 5 Modified from the standard pdb.Pdb class to avoid including readline, so that
6 6 the command line completion of other programs which include this isn't
7 7 damaged.
8 8
9 9 In the future, this class will be expanded with improvements over the standard
10 10 pdb.
11 11
12 12 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
13 13 changes. Licensing should therefore be under the standard Python terms. For
14 14 details on the PSF (Python Software Foundation) standard license, see:
15 15
16 16 http://www.python.org/2.2.3/license.html"""
17 17
18 18 #*****************************************************************************
19 19 #
20 20 # This file is licensed under the PSF license.
21 21 #
22 22 # Copyright (C) 2001 Python Software Foundation, www.python.org
23 23 # Copyright (C) 2005-2006 Fernando Perez. <fperez@colorado.edu>
24 24 #
25 25 #
26 26 #*****************************************************************************
27 27 from __future__ import print_function
28 28
29 29 import bdb
30 30 import functools
31 31 import inspect
32 32 import sys
33 33 import warnings
34 34
35 35 from IPython import get_ipython
36 36 from IPython.utils import PyColorize, ulinecache
37 37 from IPython.utils import coloransi, py3compat
38 38 from IPython.core.excolors import exception_colors
39 39 from IPython.testing.skipdoctest import skip_doctest
40 40
41 41
42 42 prompt = 'ipdb> '
43 43
44 44 #We have to check this directly from sys.argv, config struct not yet available
45 45 from pdb import Pdb as OldPdb
46 46
47 47 # Allow the set_trace code to operate outside of an ipython instance, even if
48 48 # it does so with some limitations. The rest of this support is implemented in
49 49 # the Tracer constructor.
50 50
51 51 def make_arrow(pad):
52 52 """generate the leading arrow in front of traceback or debugger"""
53 53 if pad >= 2:
54 54 return '-'*(pad-2) + '> '
55 55 elif pad == 1:
56 56 return '>'
57 57 return ''
58 58
59 59
60 60 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
61 61 """Exception hook which handles `BdbQuit` exceptions.
62 62
63 63 All other exceptions are processed using the `excepthook`
64 64 parameter.
65 65 """
66 66 warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
67 67 DeprecationWarning)
68 68 if et==bdb.BdbQuit:
69 69 print('Exiting Debugger.')
70 70 elif excepthook is not None:
71 71 excepthook(et, ev, tb)
72 72 else:
73 73 # Backwards compatibility. Raise deprecation warning?
74 74 BdbQuit_excepthook.excepthook_ori(et,ev,tb)
75 75
76 76
77 77 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
78 78 warnings.warn(
79 79 "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
80 80 DeprecationWarning)
81 81 print('Exiting Debugger.')
82 82
83 83
84 84 class Tracer(object):
85 85 """
86 86 DEPRECATED
87 87
88 88 Class for local debugging, similar to pdb.set_trace.
89 89
90 90 Instances of this class, when called, behave like pdb.set_trace, but
91 91 providing IPython's enhanced capabilities.
92 92
93 93 This is implemented as a class which must be initialized in your own code
94 94 and not as a standalone function because we need to detect at runtime
95 95 whether IPython is already active or not. That detection is done in the
96 96 constructor, ensuring that this code plays nicely with a running IPython,
97 97 while functioning acceptably (though with limitations) if outside of it.
98 98 """
99 99
100 100 @skip_doctest
101 101 def __init__(self, colors=None):
102 102 """
103 103 DEPRECATED
104 104
105 105 Create a local debugger instance.
106 106
107 107 Parameters
108 108 ----------
109 109
110 110 colors : str, optional
111 111 The name of the color scheme to use, it must be one of IPython's
112 112 valid color schemes. If not given, the function will default to
113 113 the current IPython scheme when running inside IPython, and to
114 114 'NoColor' otherwise.
115 115
116 116 Examples
117 117 --------
118 118 ::
119 119
120 120 from IPython.core.debugger import Tracer; debug_here = Tracer()
121 121
122 122 Later in your code::
123 123
124 124 debug_here() # -> will open up the debugger at that point.
125 125
126 126 Once the debugger activates, you can use all of its regular commands to
127 127 step through code, set breakpoints, etc. See the pdb documentation
128 128 from the Python standard library for usage details.
129 129 """
130 130 warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
131 131 "`IPython.core.debugger.Pdb.set_trace()`",
132 132 DeprecationWarning)
133 133
134 134 ip = get_ipython()
135 135 if ip is None:
136 136 # Outside of ipython, we set our own exception hook manually
137 137 sys.excepthook = functools.partial(BdbQuit_excepthook,
138 138 excepthook=sys.excepthook)
139 139 def_colors = 'NoColor'
140 140 else:
141 141 # In ipython, we use its custom exception handler mechanism
142 142 def_colors = ip.colors
143 143 ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
144 144
145 145 if colors is None:
146 146 colors = def_colors
147 147
148 148 # The stdlib debugger internally uses a modified repr from the `repr`
149 149 # module, that limits the length of printed strings to a hardcoded
150 150 # limit of 30 characters. That much trimming is too aggressive, let's
151 151 # at least raise that limit to 80 chars, which should be enough for
152 152 # most interactive uses.
153 153 try:
154 154 try:
155 155 from reprlib import aRepr # Py 3
156 156 except ImportError:
157 157 from repr import aRepr # Py 2
158 158 aRepr.maxstring = 80
159 159 except:
160 160 # This is only a user-facing convenience, so any error we encounter
161 161 # here can be warned about but can be otherwise ignored. These
162 162 # printouts will tell us about problems if this API changes
163 163 import traceback
164 164 traceback.print_exc()
165 165
166 166 self.debugger = Pdb(colors)
167 167
168 168 def __call__(self):
169 169 """Starts an interactive debugger at the point where called.
170 170
171 171 This is similar to the pdb.set_trace() function from the std lib, but
172 172 using IPython's enhanced debugger."""
173 173
174 174 self.debugger.set_trace(sys._getframe().f_back)
175 175
176 176
177 177 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
178 178 """Make new_fn have old_fn's doc string. This is particularly useful
179 179 for the ``do_...`` commands that hook into the help system.
180 180 Adapted from from a comp.lang.python posting
181 181 by Duncan Booth."""
182 182 def wrapper(*args, **kw):
183 183 return new_fn(*args, **kw)
184 184 if old_fn.__doc__:
185 185 wrapper.__doc__ = old_fn.__doc__ + additional_text
186 186 return wrapper
187 187
188 188
189 189 def _file_lines(fname):
190 190 """Return the contents of a named file as a list of lines.
191 191
192 192 This function never raises an IOError exception: if the file can't be
193 193 read, it simply returns an empty list."""
194 194
195 195 try:
196 196 outfile = open(fname)
197 197 except IOError:
198 198 return []
199 199 else:
200 200 out = outfile.readlines()
201 201 outfile.close()
202 202 return out
203 203
204 204
205 205 class Pdb(OldPdb, object):
206 206 """Modified Pdb class, does not load readline.
207 207
208 208 for a standalone version that uses prompt_toolkit, see
209 209 `IPython.terminal.debugger.TerminalPdb` and
210 210 `IPython.terminal.debugger.set_trace()`
211 211 """
212 212
213 213 def __init__(self, color_scheme=None, completekey=None,
214 214 stdin=None, stdout=None, context=5):
215 215
216 216 # Parent constructor:
217 217 try:
218 218 self.context = int(context)
219 219 if self.context <= 0:
220 220 raise ValueError("Context must be a positive integer")
221 221 except (TypeError, ValueError):
222 222 raise ValueError("Context must be a positive integer")
223 223
224 224 OldPdb.__init__(self, completekey, stdin, stdout)
225 225
226 226 # IPython changes...
227 227 self.shell = get_ipython()
228 228
229 229 if self.shell is None:
230 230 save_main = sys.modules['__main__']
231 231 # No IPython instance running, we must create one
232 232 from IPython.terminal.interactiveshell import \
233 233 TerminalInteractiveShell
234 234 self.shell = TerminalInteractiveShell.instance()
235 235 # needed by any code which calls __import__("__main__") after
236 236 # the debugger was entered. See also #9941.
237 237 sys.modules['__main__'] = save_main
238 238
239 239 if color_scheme is not None:
240 240 warnings.warn(
241 241 "The `color_scheme` argument is deprecated since version 5.1",
242 242 DeprecationWarning, stacklevel=2)
243 243 else:
244 244 color_scheme = self.shell.colors
245 245
246 246 self.aliases = {}
247 247
248 248 # Create color table: we copy the default one from the traceback
249 249 # module and add a few attributes needed for debugging
250 250 self.color_scheme_table = exception_colors()
251 251
252 252 # shorthands
253 253 C = coloransi.TermColors
254 254 cst = self.color_scheme_table
255 255
256 256 cst['NoColor'].colors.prompt = C.NoColor
257 257 cst['NoColor'].colors.breakpoint_enabled = C.NoColor
258 258 cst['NoColor'].colors.breakpoint_disabled = C.NoColor
259 259
260 260 cst['Linux'].colors.prompt = C.Green
261 261 cst['Linux'].colors.breakpoint_enabled = C.LightRed
262 262 cst['Linux'].colors.breakpoint_disabled = C.Red
263 263
264 264 cst['LightBG'].colors.prompt = C.Blue
265 265 cst['LightBG'].colors.breakpoint_enabled = C.LightRed
266 266 cst['LightBG'].colors.breakpoint_disabled = C.Red
267 267
268 268 cst['Neutral'].colors.prompt = C.Blue
269 269 cst['Neutral'].colors.breakpoint_enabled = C.LightRed
270 270 cst['Neutral'].colors.breakpoint_disabled = C.Red
271 271
272 272
273 273 # Add a python parser so we can syntax highlight source while
274 274 # debugging.
275 275 self.parser = PyColorize.Parser(style=color_scheme)
276 276 self.set_colors(color_scheme)
277 277
278 278 # Set the prompt - the default prompt is '(Pdb)'
279 279 self.prompt = prompt
280 280
281 281 def set_colors(self, scheme):
282 282 """Shorthand access to the color table scheme selector method."""
283 283 self.color_scheme_table.set_active_scheme(scheme)
284 284 self.parser.style = scheme
285 285
286 286 def interaction(self, frame, traceback):
287 287 try:
288 288 OldPdb.interaction(self, frame, traceback)
289 289 except KeyboardInterrupt:
290 290 sys.stdout.write('\n' + self.shell.get_exception_only())
291 291
292 292 def parseline(self, line):
293 293 if line.startswith("!!"):
294 294 # Force standard behavior.
295 295 return super(Pdb, self).parseline(line[2:])
296 296 # "Smart command mode" from pdb++: don't execute commands if a variable
297 297 # with the same name exists.
298 298 cmd, arg, newline = super(Pdb, self).parseline(line)
299 299 # Fix for #9611: Do not trigger smart command if the command is `exit`
300 300 # or `quit` and it would resolve to their *global* value (the
301 301 # `ExitAutocall` object). Just checking that it is not present in the
302 302 # locals dict is not enough as locals and globals match at the
303 303 # toplevel.
304 304 if ((cmd in self.curframe.f_locals or cmd in self.curframe.f_globals)
305 305 and not (cmd in ["exit", "quit"]
306 306 and (self.curframe.f_locals is self.curframe.f_globals
307 307 or cmd not in self.curframe.f_locals))):
308 308 return super(Pdb, self).parseline("!" + line)
309 309 return super(Pdb, self).parseline(line)
310 310
311 311 def new_do_up(self, arg):
312 312 OldPdb.do_up(self, arg)
313 313 do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
314 314
315 315 def new_do_down(self, arg):
316 316 OldPdb.do_down(self, arg)
317 317
318 318 do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
319 319
320 320 def new_do_frame(self, arg):
321 321 OldPdb.do_frame(self, arg)
322 322
323 323 def new_do_quit(self, arg):
324 324
325 325 if hasattr(self, 'old_all_completions'):
326 326 self.shell.Completer.all_completions=self.old_all_completions
327 327
328 328 return OldPdb.do_quit(self, arg)
329 329
330 330 do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
331 331
332 332 def new_do_restart(self, arg):
333 333 """Restart command. In the context of ipython this is exactly the same
334 334 thing as 'quit'."""
335 335 self.msg("Restart doesn't make sense here. Using 'quit' instead.")
336 336 return self.do_quit(arg)
337 337
338 338 def print_stack_trace(self, context=None):
339 339 if context is None:
340 340 context = self.context
341 341 try:
342 342 context=int(context)
343 343 if context <= 0:
344 344 raise ValueError("Context must be a positive integer")
345 345 except (TypeError, ValueError):
346 346 raise ValueError("Context must be a positive integer")
347 347 try:
348 348 for frame_lineno in self.stack:
349 349 self.print_stack_entry(frame_lineno, context=context)
350 350 except KeyboardInterrupt:
351 351 pass
352 352
353 353 def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
354 354 context=None):
355 355 if context is None:
356 356 context = self.context
357 357 try:
358 358 context=int(context)
359 359 if context <= 0:
360 360 raise ValueError("Context must be a positive integer")
361 361 except (TypeError, ValueError):
362 362 raise ValueError("Context must be a positive integer")
363 363 print(self.format_stack_entry(frame_lineno, '', context))
364 364
365 365 # vds: >>
366 366 frame, lineno = frame_lineno
367 367 filename = frame.f_code.co_filename
368 368 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
369 369 # vds: <<
370 370
371 371 def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
372 372 if context is None:
373 373 context = self.context
374 374 try:
375 375 context=int(context)
376 376 if context <= 0:
377 377 print("Context must be a positive integer")
378 378 except (TypeError, ValueError):
379 379 print("Context must be a positive integer")
380 380 try:
381 381 import reprlib # Py 3
382 382 except ImportError:
383 383 import repr as reprlib # Py 2
384 384
385 385 ret = []
386 386
387 387 Colors = self.color_scheme_table.active_colors
388 388 ColorsNormal = Colors.Normal
389 389 tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
390 390 tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
391 391 tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
392 392 tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
393 393 ColorsNormal)
394 394
395 395 frame, lineno = frame_lineno
396 396
397 397 return_value = ''
398 398 if '__return__' in frame.f_locals:
399 399 rv = frame.f_locals['__return__']
400 400 #return_value += '->'
401 401 return_value += reprlib.repr(rv) + '\n'
402 402 ret.append(return_value)
403 403
404 404 #s = filename + '(' + `lineno` + ')'
405 405 filename = self.canonic(frame.f_code.co_filename)
406 406 link = tpl_link % py3compat.cast_unicode(filename)
407 407
408 408 if frame.f_code.co_name:
409 409 func = frame.f_code.co_name
410 410 else:
411 411 func = "<lambda>"
412 412
413 413 call = ''
414 414 if func != '?':
415 415 if '__args__' in frame.f_locals:
416 416 args = reprlib.repr(frame.f_locals['__args__'])
417 417 else:
418 418 args = '()'
419 419 call = tpl_call % (func, args)
420 420
421 421 # The level info should be generated in the same format pdb uses, to
422 422 # avoid breaking the pdbtrack functionality of python-mode in *emacs.
423 423 if frame is self.curframe:
424 424 ret.append('> ')
425 425 else:
426 426 ret.append(' ')
427 427 ret.append(u'%s(%s)%s\n' % (link,lineno,call))
428 428
429 429 start = lineno - 1 - context//2
430 430 lines = ulinecache.getlines(filename)
431 431 start = min(start, len(lines) - context)
432 432 start = max(start, 0)
433 433 lines = lines[start : start + context]
434 434
435 435 for i,line in enumerate(lines):
436 436 show_arrow = (start + 1 + i == lineno)
437 437 linetpl = (frame is self.curframe or show_arrow) \
438 438 and tpl_line_em \
439 439 or tpl_line
440 440 ret.append(self.__format_line(linetpl, filename,
441 441 start + 1 + i, line,
442 442 arrow = show_arrow) )
443 443 return ''.join(ret)
444 444
445 445 def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
446 446 bp_mark = ""
447 447 bp_mark_color = ""
448 448
449 449 new_line, err = self.parser.format2(line, 'str')
450 450 if not err:
451 451 line = new_line
452 452
453 453 bp = None
454 454 if lineno in self.get_file_breaks(filename):
455 455 bps = self.get_breaks(filename, lineno)
456 456 bp = bps[-1]
457 457
458 458 if bp:
459 459 Colors = self.color_scheme_table.active_colors
460 460 bp_mark = str(bp.number)
461 461 bp_mark_color = Colors.breakpoint_enabled
462 462 if not bp.enabled:
463 463 bp_mark_color = Colors.breakpoint_disabled
464 464
465 465 numbers_width = 7
466 466 if arrow:
467 467 # This is the line with the error
468 468 pad = numbers_width - len(str(lineno)) - len(bp_mark)
469 469 num = '%s%s' % (make_arrow(pad), str(lineno))
470 470 else:
471 471 num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
472 472
473 473 return tpl_line % (bp_mark_color + bp_mark, num, line)
474 474
475 475
476 476 def print_list_lines(self, filename, first, last):
477 477 """The printing (as opposed to the parsing part of a 'list'
478 478 command."""
479 479 try:
480 480 Colors = self.color_scheme_table.active_colors
481 481 ColorsNormal = Colors.Normal
482 482 tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
483 483 tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
484 484 src = []
485 485 if filename == "<string>" and hasattr(self, "_exec_filename"):
486 486 filename = self._exec_filename
487 487
488 488 for lineno in range(first, last+1):
489 489 line = ulinecache.getline(filename, lineno)
490 490 if not line:
491 491 break
492 492
493 493 if lineno == self.curframe.f_lineno:
494 494 line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
495 495 else:
496 496 line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
497 497
498 498 src.append(line)
499 499 self.lineno = lineno
500 500
501 501 print(''.join(src))
502 502
503 503 except KeyboardInterrupt:
504 504 pass
505 505
506 506 def do_list(self, arg):
507 507 self.lastcmd = 'list'
508 508 last = None
509 509 if arg:
510 510 try:
511 511 x = eval(arg, {}, {})
512 512 if type(x) == type(()):
513 513 first, last = x
514 514 first = int(first)
515 515 last = int(last)
516 516 if last < first:
517 517 # Assume it's a count
518 518 last = first + last
519 519 else:
520 520 first = max(1, int(x) - 5)
521 521 except:
522 522 print('*** Error in argument:', repr(arg))
523 523 return
524 524 elif self.lineno is None:
525 525 first = max(1, self.curframe.f_lineno - 5)
526 526 else:
527 527 first = self.lineno + 1
528 528 if last is None:
529 529 last = first + 10
530 530 self.print_list_lines(self.curframe.f_code.co_filename, first, last)
531 531
532 532 # vds: >>
533 533 lineno = first
534 534 filename = self.curframe.f_code.co_filename
535 535 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
536 536 # vds: <<
537 537
538 538 do_l = do_list
539 539
540 540 def getsourcelines(self, obj):
541 541 lines, lineno = inspect.findsource(obj)
542 542 if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
543 543 # must be a module frame: do not try to cut a block out of it
544 544 return lines, 1
545 545 elif inspect.ismodule(obj):
546 546 return lines, 1
547 547 return inspect.getblock(lines[lineno:]), lineno+1
548 548
549 549 def do_longlist(self, arg):
550 550 self.lastcmd = 'longlist'
551 551 try:
552 552 lines, lineno = self.getsourcelines(self.curframe)
553 553 except OSError as err:
554 554 self.error(err)
555 555 return
556 556 last = lineno + len(lines)
557 557 self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
558 558 do_ll = do_longlist
559 559
560 560 def do_pdef(self, arg):
561 561 """Print the call signature for any callable object.
562 562
563 563 The debugger interface to %pdef"""
564 564 namespaces = [('Locals', self.curframe.f_locals),
565 565 ('Globals', self.curframe.f_globals)]
566 566 self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
567 567
568 568 def do_pdoc(self, arg):
569 569 """Print the docstring for an object.
570 570
571 571 The debugger interface to %pdoc."""
572 572 namespaces = [('Locals', self.curframe.f_locals),
573 573 ('Globals', self.curframe.f_globals)]
574 574 self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
575 575
576 576 def do_pfile(self, arg):
577 577 """Print (or run through pager) the file where an object is defined.
578 578
579 579 The debugger interface to %pfile.
580 580 """
581 581 namespaces = [('Locals', self.curframe.f_locals),
582 582 ('Globals', self.curframe.f_globals)]
583 583 self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
584 584
585 585 def do_pinfo(self, arg):
586 586 """Provide detailed information about an object.
587 587
588 588 The debugger interface to %pinfo, i.e., obj?."""
589 589 namespaces = [('Locals', self.curframe.f_locals),
590 590 ('Globals', self.curframe.f_globals)]
591 591 self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
592 592
593 593 def do_pinfo2(self, arg):
594 594 """Provide extra detailed information about an object.
595 595
596 596 The debugger interface to %pinfo2, i.e., obj??."""
597 597 namespaces = [('Locals', self.curframe.f_locals),
598 598 ('Globals', self.curframe.f_globals)]
599 599 self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
600 600
601 601 def do_psource(self, arg):
602 602 """Print (or run through pager) the source code for an object."""
603 603 namespaces = [('Locals', self.curframe.f_locals),
604 604 ('Globals', self.curframe.f_globals)]
605 605 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
606 606
607 if sys.version_info > (3, ):
608 def do_where(self, arg):
609 """w(here)
610 Print a stack trace, with the most recent frame at the bottom.
611 An arrow indicates the "current frame", which determines the
612 context of most commands. 'bt' is an alias for this command.
613
614 Take a number as argument as an (optional) number of context line to
615 print"""
616 if arg:
617 context = int(arg)
618 self.print_stack_trace(context)
619 else:
620 self.print_stack_trace()
607 def do_where(self, arg):
608 """w(here)
609 Print a stack trace, with the most recent frame at the bottom.
610 An arrow indicates the "current frame", which determines the
611 context of most commands. 'bt' is an alias for this command.
612
613 Take a number as argument as an (optional) number of context line to
614 print"""
615 if arg:
616 context = int(arg)
617 self.print_stack_trace(context)
618 else:
619 self.print_stack_trace()
621 620
622 do_w = do_where
621 do_w = do_where
623 622
624 623
625 624 def set_trace(frame=None):
626 625 """
627 626 Start debugging from `frame`.
628 627
629 628 If frame is not specified, debugging starts from caller's frame.
630 629 """
631 630 Pdb().set_trace(frame or sys._getframe().f_back)
@@ -1,3230 +1,3227 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13 from __future__ import absolute_import, print_function
14 14
15 15 import __future__
16 16 import abc
17 17 import ast
18 18 import atexit
19 19 import functools
20 20 import os
21 21 import re
22 22 import runpy
23 23 import sys
24 24 import tempfile
25 25 import traceback
26 26 import types
27 27 import subprocess
28 28 import warnings
29 29 from io import open as io_open
30 30
31 31 from pickleshare import PickleShareDB
32 32
33 33 from traitlets.config.configurable import SingletonConfigurable
34 34 from IPython.core import oinspect
35 35 from IPython.core import magic
36 36 from IPython.core import page
37 37 from IPython.core import prefilter
38 38 from IPython.core import shadowns
39 39 from IPython.core import ultratb
40 40 from IPython.core.alias import Alias, AliasManager
41 41 from IPython.core.autocall import ExitAutocall
42 42 from IPython.core.builtin_trap import BuiltinTrap
43 43 from IPython.core.events import EventManager, available_events
44 44 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
45 45 from IPython.core.debugger import Pdb
46 46 from IPython.core.display_trap import DisplayTrap
47 47 from IPython.core.displayhook import DisplayHook
48 48 from IPython.core.displaypub import DisplayPublisher
49 49 from IPython.core.error import InputRejected, UsageError
50 50 from IPython.core.extensions import ExtensionManager
51 51 from IPython.core.formatters import DisplayFormatter
52 52 from IPython.core.history import HistoryManager
53 53 from IPython.core.inputsplitter import ESC_MAGIC, ESC_MAGIC2
54 54 from IPython.core.logger import Logger
55 55 from IPython.core.macro import Macro
56 56 from IPython.core.payload import PayloadManager
57 57 from IPython.core.prefilter import PrefilterManager
58 58 from IPython.core.profiledir import ProfileDir
59 59 from IPython.core.usage import default_banner
60 60 from IPython.testing.skipdoctest import skip_doctest_py2, skip_doctest
61 61 from IPython.utils import PyColorize
62 62 from IPython.utils import io
63 63 from IPython.utils import py3compat
64 64 from IPython.utils import openpy
65 65 from IPython.utils.decorators import undoc
66 66 from IPython.utils.io import ask_yes_no
67 67 from IPython.utils.ipstruct import Struct
68 68 from IPython.paths import get_ipython_dir
69 69 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
70 70 from IPython.utils.process import system, getoutput
71 71 from IPython.utils.py3compat import (builtin_mod, unicode_type, string_types,
72 72 with_metaclass, iteritems)
73 73 from IPython.utils.strdispatch import StrDispatch
74 74 from IPython.utils.syspathcontext import prepended_to_syspath
75 75 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
76 76 from IPython.utils.tempdir import TemporaryDirectory
77 77 from traitlets import (
78 78 Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
79 79 observe, default,
80 80 )
81 81 from warnings import warn
82 82 from logging import error
83 83 import IPython.core.hooks
84 84
85 85 # NoOpContext is deprecated, but ipykernel imports it from here.
86 86 # See https://github.com/ipython/ipykernel/issues/157
87 87 from IPython.utils.contexts import NoOpContext
88 88
89 89 try:
90 90 import docrepr.sphinxify as sphx
91 91
92 92 def sphinxify(doc):
93 93 with TemporaryDirectory() as dirname:
94 94 return {
95 95 'text/html': sphx.sphinxify(doc, dirname),
96 96 'text/plain': doc
97 97 }
98 98 except ImportError:
99 99 sphinxify = None
100 100
101 101
102 102 class ProvisionalWarning(DeprecationWarning):
103 103 """
104 104 Warning class for unstable features
105 105 """
106 106 pass
107 107
108 108 #-----------------------------------------------------------------------------
109 109 # Globals
110 110 #-----------------------------------------------------------------------------
111 111
112 112 # compiled regexps for autoindent management
113 113 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
114 114
115 115 #-----------------------------------------------------------------------------
116 116 # Utilities
117 117 #-----------------------------------------------------------------------------
118 118
119 119 @undoc
120 120 def softspace(file, newvalue):
121 121 """Copied from code.py, to remove the dependency"""
122 122
123 123 oldvalue = 0
124 124 try:
125 125 oldvalue = file.softspace
126 126 except AttributeError:
127 127 pass
128 128 try:
129 129 file.softspace = newvalue
130 130 except (AttributeError, TypeError):
131 131 # "attribute-less object" or "read-only attributes"
132 132 pass
133 133 return oldvalue
134 134
135 135 @undoc
136 136 def no_op(*a, **kw): pass
137 137
138 138
139 139 class SpaceInInput(Exception): pass
140 140
141 141
142 142 def get_default_colors():
143 143 "DEPRECATED"
144 144 warn('get_default_color is Deprecated, and is `Neutral` on all platforms.',
145 145 DeprecationWarning, stacklevel=2)
146 146 return 'Neutral'
147 147
148 148
149 149 class SeparateUnicode(Unicode):
150 150 r"""A Unicode subclass to validate separate_in, separate_out, etc.
151 151
152 152 This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
153 153 """
154 154
155 155 def validate(self, obj, value):
156 156 if value == '0': value = ''
157 157 value = value.replace('\\n','\n')
158 158 return super(SeparateUnicode, self).validate(obj, value)
159 159
160 160
161 161 @undoc
162 162 class DummyMod(object):
163 163 """A dummy module used for IPython's interactive module when
164 164 a namespace must be assigned to the module's __dict__."""
165 165 pass
166 166
167 167
168 168 class ExecutionResult(object):
169 169 """The result of a call to :meth:`InteractiveShell.run_cell`
170 170
171 171 Stores information about what took place.
172 172 """
173 173 execution_count = None
174 174 error_before_exec = None
175 175 error_in_exec = None
176 176 result = None
177 177
178 178 @property
179 179 def success(self):
180 180 return (self.error_before_exec is None) and (self.error_in_exec is None)
181 181
182 182 def raise_error(self):
183 183 """Reraises error if `success` is `False`, otherwise does nothing"""
184 184 if self.error_before_exec is not None:
185 185 raise self.error_before_exec
186 186 if self.error_in_exec is not None:
187 187 raise self.error_in_exec
188 188
189 189 def __repr__(self):
190 if sys.version_info > (3,):
191 name = self.__class__.__qualname__
192 else:
193 name = self.__class__.__name__
190 name = self.__class__.__qualname__
194 191 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s result=%s>' %\
195 192 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.result))
196 193
197 194
198 195 class InteractiveShell(SingletonConfigurable):
199 196 """An enhanced, interactive shell for Python."""
200 197
201 198 _instance = None
202 199
203 200 ast_transformers = List([], help=
204 201 """
205 202 A list of ast.NodeTransformer subclass instances, which will be applied
206 203 to user input before code is run.
207 204 """
208 205 ).tag(config=True)
209 206
210 207 autocall = Enum((0,1,2), default_value=0, help=
211 208 """
212 209 Make IPython automatically call any callable object even if you didn't
213 210 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
214 211 automatically. The value can be '0' to disable the feature, '1' for
215 212 'smart' autocall, where it is not applied if there are no more
216 213 arguments on the line, and '2' for 'full' autocall, where all callable
217 214 objects are automatically called (even if no arguments are present).
218 215 """
219 216 ).tag(config=True)
220 217 # TODO: remove all autoindent logic and put into frontends.
221 218 # We can't do this yet because even runlines uses the autoindent.
222 219 autoindent = Bool(True, help=
223 220 """
224 221 Autoindent IPython code entered interactively.
225 222 """
226 223 ).tag(config=True)
227 224
228 225 automagic = Bool(True, help=
229 226 """
230 227 Enable magic commands to be called without the leading %.
231 228 """
232 229 ).tag(config=True)
233 230
234 231 banner1 = Unicode(default_banner,
235 232 help="""The part of the banner to be printed before the profile"""
236 233 ).tag(config=True)
237 234 banner2 = Unicode('',
238 235 help="""The part of the banner to be printed after the profile"""
239 236 ).tag(config=True)
240 237
241 238 cache_size = Integer(1000, help=
242 239 """
243 240 Set the size of the output cache. The default is 1000, you can
244 241 change it permanently in your config file. Setting it to 0 completely
245 242 disables the caching system, and the minimum value accepted is 20 (if
246 243 you provide a value less than 20, it is reset to 0 and a warning is
247 244 issued). This limit is defined because otherwise you'll spend more
248 245 time re-flushing a too small cache than working
249 246 """
250 247 ).tag(config=True)
251 248 color_info = Bool(True, help=
252 249 """
253 250 Use colors for displaying information about objects. Because this
254 251 information is passed through a pager (like 'less'), and some pagers
255 252 get confused with color codes, this capability can be turned off.
256 253 """
257 254 ).tag(config=True)
258 255 colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
259 256 default_value='Neutral',
260 257 help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
261 258 ).tag(config=True)
262 259 debug = Bool(False).tag(config=True)
263 260 disable_failing_post_execute = Bool(False,
264 261 help="Don't call post-execute functions that have failed in the past."
265 262 ).tag(config=True)
266 263 display_formatter = Instance(DisplayFormatter, allow_none=True)
267 264 displayhook_class = Type(DisplayHook)
268 265 display_pub_class = Type(DisplayPublisher)
269 266
270 267 sphinxify_docstring = Bool(False, help=
271 268 """
272 269 Enables rich html representation of docstrings. (This requires the
273 270 docrepr module).
274 271 """).tag(config=True)
275 272
276 273 @observe("sphinxify_docstring")
277 274 def _sphinxify_docstring_changed(self, change):
278 275 if change['new']:
279 276 warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
280 277
281 278 enable_html_pager = Bool(False, help=
282 279 """
283 280 (Provisional API) enables html representation in mime bundles sent
284 281 to pagers.
285 282 """).tag(config=True)
286 283
287 284 @observe("enable_html_pager")
288 285 def _enable_html_pager_changed(self, change):
289 286 if change['new']:
290 287 warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
291 288
292 289 data_pub_class = None
293 290
294 291 exit_now = Bool(False)
295 292 exiter = Instance(ExitAutocall)
296 293 @default('exiter')
297 294 def _exiter_default(self):
298 295 return ExitAutocall(self)
299 296 # Monotonically increasing execution counter
300 297 execution_count = Integer(1)
301 298 filename = Unicode("<ipython console>")
302 299 ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
303 300
304 301 # Input splitter, to transform input line by line and detect when a block
305 302 # is ready to be executed.
306 303 input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
307 304 (), {'line_input_checker': True})
308 305
309 306 # This InputSplitter instance is used to transform completed cells before
310 307 # running them. It allows cell magics to contain blank lines.
311 308 input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
312 309 (), {'line_input_checker': False})
313 310
314 311 logstart = Bool(False, help=
315 312 """
316 313 Start logging to the default log file in overwrite mode.
317 314 Use `logappend` to specify a log file to **append** logs to.
318 315 """
319 316 ).tag(config=True)
320 317 logfile = Unicode('', help=
321 318 """
322 319 The name of the logfile to use.
323 320 """
324 321 ).tag(config=True)
325 322 logappend = Unicode('', help=
326 323 """
327 324 Start logging to the given file in append mode.
328 325 Use `logfile` to specify a log file to **overwrite** logs to.
329 326 """
330 327 ).tag(config=True)
331 328 object_info_string_level = Enum((0,1,2), default_value=0,
332 329 ).tag(config=True)
333 330 pdb = Bool(False, help=
334 331 """
335 332 Automatically call the pdb debugger after every exception.
336 333 """
337 334 ).tag(config=True)
338 335 display_page = Bool(False,
339 336 help="""If True, anything that would be passed to the pager
340 337 will be displayed as regular output instead."""
341 338 ).tag(config=True)
342 339
343 340 # deprecated prompt traits:
344 341
345 342 prompt_in1 = Unicode('In [\\#]: ',
346 343 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
347 344 ).tag(config=True)
348 345 prompt_in2 = Unicode(' .\\D.: ',
349 346 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
350 347 ).tag(config=True)
351 348 prompt_out = Unicode('Out[\\#]: ',
352 349 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
353 350 ).tag(config=True)
354 351 prompts_pad_left = Bool(True,
355 352 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
356 353 ).tag(config=True)
357 354
358 355 @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
359 356 def _prompt_trait_changed(self, change):
360 357 name = change['name']
361 358 warn("InteractiveShell.{name} is deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly.".format(
362 359 name=name)
363 360 )
364 361 # protect against weird cases where self.config may not exist:
365 362
366 363 show_rewritten_input = Bool(True,
367 364 help="Show rewritten input, e.g. for autocall."
368 365 ).tag(config=True)
369 366
370 367 quiet = Bool(False).tag(config=True)
371 368
372 369 history_length = Integer(10000,
373 370 help='Total length of command history'
374 371 ).tag(config=True)
375 372
376 373 history_load_length = Integer(1000, help=
377 374 """
378 375 The number of saved history entries to be loaded
379 376 into the history buffer at startup.
380 377 """
381 378 ).tag(config=True)
382 379
383 380 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none'],
384 381 default_value='last_expr',
385 382 help="""
386 383 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
387 384 run interactively (displaying output from expressions)."""
388 385 ).tag(config=True)
389 386
390 387 # TODO: this part of prompt management should be moved to the frontends.
391 388 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
392 389 separate_in = SeparateUnicode('\n').tag(config=True)
393 390 separate_out = SeparateUnicode('').tag(config=True)
394 391 separate_out2 = SeparateUnicode('').tag(config=True)
395 392 wildcards_case_sensitive = Bool(True).tag(config=True)
396 393 xmode = CaselessStrEnum(('Context','Plain', 'Verbose'),
397 394 default_value='Context').tag(config=True)
398 395
399 396 # Subcomponents of InteractiveShell
400 397 alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
401 398 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
402 399 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
403 400 display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
404 401 extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
405 402 payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
406 403 history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
407 404 magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
408 405
409 406 profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
410 407 @property
411 408 def profile(self):
412 409 if self.profile_dir is not None:
413 410 name = os.path.basename(self.profile_dir.location)
414 411 return name.replace('profile_','')
415 412
416 413
417 414 # Private interface
418 415 _post_execute = Dict()
419 416
420 417 # Tracks any GUI loop loaded for pylab
421 418 pylab_gui_select = None
422 419
423 420 last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
424 421
425 422 def __init__(self, ipython_dir=None, profile_dir=None,
426 423 user_module=None, user_ns=None,
427 424 custom_exceptions=((), None), **kwargs):
428 425
429 426 # This is where traits with a config_key argument are updated
430 427 # from the values on config.
431 428 super(InteractiveShell, self).__init__(**kwargs)
432 429 if 'PromptManager' in self.config:
433 430 warn('As of IPython 5.0 `PromptManager` config will have no effect'
434 431 ' and has been replaced by TerminalInteractiveShell.prompts_class')
435 432 self.configurables = [self]
436 433
437 434 # These are relatively independent and stateless
438 435 self.init_ipython_dir(ipython_dir)
439 436 self.init_profile_dir(profile_dir)
440 437 self.init_instance_attrs()
441 438 self.init_environment()
442 439
443 440 # Check if we're in a virtualenv, and set up sys.path.
444 441 self.init_virtualenv()
445 442
446 443 # Create namespaces (user_ns, user_global_ns, etc.)
447 444 self.init_create_namespaces(user_module, user_ns)
448 445 # This has to be done after init_create_namespaces because it uses
449 446 # something in self.user_ns, but before init_sys_modules, which
450 447 # is the first thing to modify sys.
451 448 # TODO: When we override sys.stdout and sys.stderr before this class
452 449 # is created, we are saving the overridden ones here. Not sure if this
453 450 # is what we want to do.
454 451 self.save_sys_module_state()
455 452 self.init_sys_modules()
456 453
457 454 # While we're trying to have each part of the code directly access what
458 455 # it needs without keeping redundant references to objects, we have too
459 456 # much legacy code that expects ip.db to exist.
460 457 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
461 458
462 459 self.init_history()
463 460 self.init_encoding()
464 461 self.init_prefilter()
465 462
466 463 self.init_syntax_highlighting()
467 464 self.init_hooks()
468 465 self.init_events()
469 466 self.init_pushd_popd_magic()
470 467 self.init_user_ns()
471 468 self.init_logger()
472 469 self.init_builtins()
473 470
474 471 # The following was in post_config_initialization
475 472 self.init_inspector()
476 473 self.raw_input_original = input
477 474 self.init_completer()
478 475 # TODO: init_io() needs to happen before init_traceback handlers
479 476 # because the traceback handlers hardcode the stdout/stderr streams.
480 477 # This logic in in debugger.Pdb and should eventually be changed.
481 478 self.init_io()
482 479 self.init_traceback_handlers(custom_exceptions)
483 480 self.init_prompts()
484 481 self.init_display_formatter()
485 482 self.init_display_pub()
486 483 self.init_data_pub()
487 484 self.init_displayhook()
488 485 self.init_magics()
489 486 self.init_alias()
490 487 self.init_logstart()
491 488 self.init_pdb()
492 489 self.init_extension_manager()
493 490 self.init_payload()
494 491 self.init_deprecation_warnings()
495 492 self.hooks.late_startup_hook()
496 493 self.events.trigger('shell_initialized', self)
497 494 atexit.register(self.atexit_operations)
498 495
499 496 def get_ipython(self):
500 497 """Return the currently running IPython instance."""
501 498 return self
502 499
503 500 #-------------------------------------------------------------------------
504 501 # Trait changed handlers
505 502 #-------------------------------------------------------------------------
506 503 @observe('ipython_dir')
507 504 def _ipython_dir_changed(self, change):
508 505 ensure_dir_exists(change['new'])
509 506
510 507 def set_autoindent(self,value=None):
511 508 """Set the autoindent flag.
512 509
513 510 If called with no arguments, it acts as a toggle."""
514 511 if value is None:
515 512 self.autoindent = not self.autoindent
516 513 else:
517 514 self.autoindent = value
518 515
519 516 #-------------------------------------------------------------------------
520 517 # init_* methods called by __init__
521 518 #-------------------------------------------------------------------------
522 519
523 520 def init_ipython_dir(self, ipython_dir):
524 521 if ipython_dir is not None:
525 522 self.ipython_dir = ipython_dir
526 523 return
527 524
528 525 self.ipython_dir = get_ipython_dir()
529 526
530 527 def init_profile_dir(self, profile_dir):
531 528 if profile_dir is not None:
532 529 self.profile_dir = profile_dir
533 530 return
534 531 self.profile_dir =\
535 532 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
536 533
537 534 def init_instance_attrs(self):
538 535 self.more = False
539 536
540 537 # command compiler
541 538 self.compile = CachingCompiler()
542 539
543 540 # Make an empty namespace, which extension writers can rely on both
544 541 # existing and NEVER being used by ipython itself. This gives them a
545 542 # convenient location for storing additional information and state
546 543 # their extensions may require, without fear of collisions with other
547 544 # ipython names that may develop later.
548 545 self.meta = Struct()
549 546
550 547 # Temporary files used for various purposes. Deleted at exit.
551 548 self.tempfiles = []
552 549 self.tempdirs = []
553 550
554 551 # keep track of where we started running (mainly for crash post-mortem)
555 552 # This is not being used anywhere currently.
556 553 self.starting_dir = py3compat.getcwd()
557 554
558 555 # Indentation management
559 556 self.indent_current_nsp = 0
560 557
561 558 # Dict to track post-execution functions that have been registered
562 559 self._post_execute = {}
563 560
564 561 def init_environment(self):
565 562 """Any changes we need to make to the user's environment."""
566 563 pass
567 564
568 565 def init_encoding(self):
569 566 # Get system encoding at startup time. Certain terminals (like Emacs
570 567 # under Win32 have it set to None, and we need to have a known valid
571 568 # encoding to use in the raw_input() method
572 569 try:
573 570 self.stdin_encoding = sys.stdin.encoding or 'ascii'
574 571 except AttributeError:
575 572 self.stdin_encoding = 'ascii'
576 573
577 574
578 575 @observe('colors')
579 576 def init_syntax_highlighting(self, changes=None):
580 577 # Python source parser/formatter for syntax highlighting
581 578 pyformat = PyColorize.Parser(style=self.colors, parent=self).format
582 579 self.pycolorize = lambda src: pyformat(src,'str')
583 580
584 581 def refresh_style(self):
585 582 # No-op here, used in subclass
586 583 pass
587 584
588 585 def init_pushd_popd_magic(self):
589 586 # for pushd/popd management
590 587 self.home_dir = get_home_dir()
591 588
592 589 self.dir_stack = []
593 590
594 591 def init_logger(self):
595 592 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
596 593 logmode='rotate')
597 594
598 595 def init_logstart(self):
599 596 """Initialize logging in case it was requested at the command line.
600 597 """
601 598 if self.logappend:
602 599 self.magic('logstart %s append' % self.logappend)
603 600 elif self.logfile:
604 601 self.magic('logstart %s' % self.logfile)
605 602 elif self.logstart:
606 603 self.magic('logstart')
607 604
608 605 def init_deprecation_warnings(self):
609 606 """
610 607 register default filter for deprecation warning.
611 608
612 609 This will allow deprecation warning of function used interactively to show
613 610 warning to users, and still hide deprecation warning from libraries import.
614 611 """
615 612 warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
616 613
617 614 def init_builtins(self):
618 615 # A single, static flag that we set to True. Its presence indicates
619 616 # that an IPython shell has been created, and we make no attempts at
620 617 # removing on exit or representing the existence of more than one
621 618 # IPython at a time.
622 619 builtin_mod.__dict__['__IPYTHON__'] = True
623 620
624 621 self.builtin_trap = BuiltinTrap(shell=self)
625 622
626 623 def init_inspector(self):
627 624 # Object inspector
628 625 self.inspector = oinspect.Inspector(oinspect.InspectColors,
629 626 PyColorize.ANSICodeColors,
630 627 'NoColor',
631 628 self.object_info_string_level)
632 629
633 630 def init_io(self):
634 631 # This will just use sys.stdout and sys.stderr. If you want to
635 632 # override sys.stdout and sys.stderr themselves, you need to do that
636 633 # *before* instantiating this class, because io holds onto
637 634 # references to the underlying streams.
638 635 # io.std* are deprecated, but don't show our own deprecation warnings
639 636 # during initialization of the deprecated API.
640 637 with warnings.catch_warnings():
641 638 warnings.simplefilter('ignore', DeprecationWarning)
642 639 io.stdout = io.IOStream(sys.stdout)
643 640 io.stderr = io.IOStream(sys.stderr)
644 641
645 642 def init_prompts(self):
646 643 # Set system prompts, so that scripts can decide if they are running
647 644 # interactively.
648 645 sys.ps1 = 'In : '
649 646 sys.ps2 = '...: '
650 647 sys.ps3 = 'Out: '
651 648
652 649 def init_display_formatter(self):
653 650 self.display_formatter = DisplayFormatter(parent=self)
654 651 self.configurables.append(self.display_formatter)
655 652
656 653 def init_display_pub(self):
657 654 self.display_pub = self.display_pub_class(parent=self)
658 655 self.configurables.append(self.display_pub)
659 656
660 657 def init_data_pub(self):
661 658 if not self.data_pub_class:
662 659 self.data_pub = None
663 660 return
664 661 self.data_pub = self.data_pub_class(parent=self)
665 662 self.configurables.append(self.data_pub)
666 663
667 664 def init_displayhook(self):
668 665 # Initialize displayhook, set in/out prompts and printing system
669 666 self.displayhook = self.displayhook_class(
670 667 parent=self,
671 668 shell=self,
672 669 cache_size=self.cache_size,
673 670 )
674 671 self.configurables.append(self.displayhook)
675 672 # This is a context manager that installs/revmoes the displayhook at
676 673 # the appropriate time.
677 674 self.display_trap = DisplayTrap(hook=self.displayhook)
678 675
679 676 def init_virtualenv(self):
680 677 """Add a virtualenv to sys.path so the user can import modules from it.
681 678 This isn't perfect: it doesn't use the Python interpreter with which the
682 679 virtualenv was built, and it ignores the --no-site-packages option. A
683 680 warning will appear suggesting the user installs IPython in the
684 681 virtualenv, but for many cases, it probably works well enough.
685 682
686 683 Adapted from code snippets online.
687 684
688 685 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
689 686 """
690 687 if 'VIRTUAL_ENV' not in os.environ:
691 688 # Not in a virtualenv
692 689 return
693 690
694 691 # venv detection:
695 692 # stdlib venv may symlink sys.executable, so we can't use realpath.
696 693 # but others can symlink *to* the venv Python, so we can't just use sys.executable.
697 694 # So we just check every item in the symlink tree (generally <= 3)
698 695 p = os.path.normcase(sys.executable)
699 696 paths = [p]
700 697 while os.path.islink(p):
701 698 p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
702 699 paths.append(p)
703 700 p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
704 701 if any(p.startswith(p_venv) for p in paths):
705 702 # Running properly in the virtualenv, don't need to do anything
706 703 return
707 704
708 705 warn("Attempting to work in a virtualenv. If you encounter problems, please "
709 706 "install IPython inside the virtualenv.")
710 707 if sys.platform == "win32":
711 708 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
712 709 else:
713 710 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
714 711 'python%d.%d' % sys.version_info[:2], 'site-packages')
715 712
716 713 import site
717 714 sys.path.insert(0, virtual_env)
718 715 site.addsitedir(virtual_env)
719 716
720 717 #-------------------------------------------------------------------------
721 718 # Things related to injections into the sys module
722 719 #-------------------------------------------------------------------------
723 720
724 721 def save_sys_module_state(self):
725 722 """Save the state of hooks in the sys module.
726 723
727 724 This has to be called after self.user_module is created.
728 725 """
729 726 self._orig_sys_module_state = {'stdin': sys.stdin,
730 727 'stdout': sys.stdout,
731 728 'stderr': sys.stderr,
732 729 'excepthook': sys.excepthook}
733 730 self._orig_sys_modules_main_name = self.user_module.__name__
734 731 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
735 732
736 733 def restore_sys_module_state(self):
737 734 """Restore the state of the sys module."""
738 735 try:
739 736 for k, v in iteritems(self._orig_sys_module_state):
740 737 setattr(sys, k, v)
741 738 except AttributeError:
742 739 pass
743 740 # Reset what what done in self.init_sys_modules
744 741 if self._orig_sys_modules_main_mod is not None:
745 742 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
746 743
747 744 #-------------------------------------------------------------------------
748 745 # Things related to the banner
749 746 #-------------------------------------------------------------------------
750 747
751 748 @property
752 749 def banner(self):
753 750 banner = self.banner1
754 751 if self.profile and self.profile != 'default':
755 752 banner += '\nIPython profile: %s\n' % self.profile
756 753 if self.banner2:
757 754 banner += '\n' + self.banner2
758 755 return banner
759 756
760 757 def show_banner(self, banner=None):
761 758 if banner is None:
762 759 banner = self.banner
763 760 sys.stdout.write(banner)
764 761
765 762 #-------------------------------------------------------------------------
766 763 # Things related to hooks
767 764 #-------------------------------------------------------------------------
768 765
769 766 def init_hooks(self):
770 767 # hooks holds pointers used for user-side customizations
771 768 self.hooks = Struct()
772 769
773 770 self.strdispatchers = {}
774 771
775 772 # Set all default hooks, defined in the IPython.hooks module.
776 773 hooks = IPython.core.hooks
777 774 for hook_name in hooks.__all__:
778 775 # default hooks have priority 100, i.e. low; user hooks should have
779 776 # 0-100 priority
780 777 self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
781 778
782 779 if self.display_page:
783 780 self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
784 781
785 782 def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
786 783 _warn_deprecated=True):
787 784 """set_hook(name,hook) -> sets an internal IPython hook.
788 785
789 786 IPython exposes some of its internal API as user-modifiable hooks. By
790 787 adding your function to one of these hooks, you can modify IPython's
791 788 behavior to call at runtime your own routines."""
792 789
793 790 # At some point in the future, this should validate the hook before it
794 791 # accepts it. Probably at least check that the hook takes the number
795 792 # of args it's supposed to.
796 793
797 794 f = types.MethodType(hook,self)
798 795
799 796 # check if the hook is for strdispatcher first
800 797 if str_key is not None:
801 798 sdp = self.strdispatchers.get(name, StrDispatch())
802 799 sdp.add_s(str_key, f, priority )
803 800 self.strdispatchers[name] = sdp
804 801 return
805 802 if re_key is not None:
806 803 sdp = self.strdispatchers.get(name, StrDispatch())
807 804 sdp.add_re(re.compile(re_key), f, priority )
808 805 self.strdispatchers[name] = sdp
809 806 return
810 807
811 808 dp = getattr(self.hooks, name, None)
812 809 if name not in IPython.core.hooks.__all__:
813 810 print("Warning! Hook '%s' is not one of %s" % \
814 811 (name, IPython.core.hooks.__all__ ))
815 812
816 813 if _warn_deprecated and (name in IPython.core.hooks.deprecated):
817 814 alternative = IPython.core.hooks.deprecated[name]
818 815 warn("Hook {} is deprecated. Use {} instead.".format(name, alternative))
819 816
820 817 if not dp:
821 818 dp = IPython.core.hooks.CommandChainDispatcher()
822 819
823 820 try:
824 821 dp.add(f,priority)
825 822 except AttributeError:
826 823 # it was not commandchain, plain old func - replace
827 824 dp = f
828 825
829 826 setattr(self.hooks,name, dp)
830 827
831 828 #-------------------------------------------------------------------------
832 829 # Things related to events
833 830 #-------------------------------------------------------------------------
834 831
835 832 def init_events(self):
836 833 self.events = EventManager(self, available_events)
837 834
838 835 self.events.register("pre_execute", self._clear_warning_registry)
839 836
840 837 def register_post_execute(self, func):
841 838 """DEPRECATED: Use ip.events.register('post_run_cell', func)
842 839
843 840 Register a function for calling after code execution.
844 841 """
845 842 warn("ip.register_post_execute is deprecated, use "
846 843 "ip.events.register('post_run_cell', func) instead.")
847 844 self.events.register('post_run_cell', func)
848 845
849 846 def _clear_warning_registry(self):
850 847 # clear the warning registry, so that different code blocks with
851 848 # overlapping line number ranges don't cause spurious suppression of
852 849 # warnings (see gh-6611 for details)
853 850 if "__warningregistry__" in self.user_global_ns:
854 851 del self.user_global_ns["__warningregistry__"]
855 852
856 853 #-------------------------------------------------------------------------
857 854 # Things related to the "main" module
858 855 #-------------------------------------------------------------------------
859 856
860 857 def new_main_mod(self, filename, modname):
861 858 """Return a new 'main' module object for user code execution.
862 859
863 860 ``filename`` should be the path of the script which will be run in the
864 861 module. Requests with the same filename will get the same module, with
865 862 its namespace cleared.
866 863
867 864 ``modname`` should be the module name - normally either '__main__' or
868 865 the basename of the file without the extension.
869 866
870 867 When scripts are executed via %run, we must keep a reference to their
871 868 __main__ module around so that Python doesn't
872 869 clear it, rendering references to module globals useless.
873 870
874 871 This method keeps said reference in a private dict, keyed by the
875 872 absolute path of the script. This way, for multiple executions of the
876 873 same script we only keep one copy of the namespace (the last one),
877 874 thus preventing memory leaks from old references while allowing the
878 875 objects from the last execution to be accessible.
879 876 """
880 877 filename = os.path.abspath(filename)
881 878 try:
882 879 main_mod = self._main_mod_cache[filename]
883 880 except KeyError:
884 881 main_mod = self._main_mod_cache[filename] = types.ModuleType(
885 882 py3compat.cast_bytes_py2(modname),
886 883 doc="Module created for script run in IPython")
887 884 else:
888 885 main_mod.__dict__.clear()
889 886 main_mod.__name__ = modname
890 887
891 888 main_mod.__file__ = filename
892 889 # It seems pydoc (and perhaps others) needs any module instance to
893 890 # implement a __nonzero__ method
894 891 main_mod.__nonzero__ = lambda : True
895 892
896 893 return main_mod
897 894
898 895 def clear_main_mod_cache(self):
899 896 """Clear the cache of main modules.
900 897
901 898 Mainly for use by utilities like %reset.
902 899
903 900 Examples
904 901 --------
905 902
906 903 In [15]: import IPython
907 904
908 905 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
909 906
910 907 In [17]: len(_ip._main_mod_cache) > 0
911 908 Out[17]: True
912 909
913 910 In [18]: _ip.clear_main_mod_cache()
914 911
915 912 In [19]: len(_ip._main_mod_cache) == 0
916 913 Out[19]: True
917 914 """
918 915 self._main_mod_cache.clear()
919 916
920 917 #-------------------------------------------------------------------------
921 918 # Things related to debugging
922 919 #-------------------------------------------------------------------------
923 920
924 921 def init_pdb(self):
925 922 # Set calling of pdb on exceptions
926 923 # self.call_pdb is a property
927 924 self.call_pdb = self.pdb
928 925
929 926 def _get_call_pdb(self):
930 927 return self._call_pdb
931 928
932 929 def _set_call_pdb(self,val):
933 930
934 931 if val not in (0,1,False,True):
935 932 raise ValueError('new call_pdb value must be boolean')
936 933
937 934 # store value in instance
938 935 self._call_pdb = val
939 936
940 937 # notify the actual exception handlers
941 938 self.InteractiveTB.call_pdb = val
942 939
943 940 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
944 941 'Control auto-activation of pdb at exceptions')
945 942
946 943 def debugger(self,force=False):
947 944 """Call the pdb debugger.
948 945
949 946 Keywords:
950 947
951 948 - force(False): by default, this routine checks the instance call_pdb
952 949 flag and does not actually invoke the debugger if the flag is false.
953 950 The 'force' option forces the debugger to activate even if the flag
954 951 is false.
955 952 """
956 953
957 954 if not (force or self.call_pdb):
958 955 return
959 956
960 957 if not hasattr(sys,'last_traceback'):
961 958 error('No traceback has been produced, nothing to debug.')
962 959 return
963 960
964 961 self.InteractiveTB.debugger(force=True)
965 962
966 963 #-------------------------------------------------------------------------
967 964 # Things related to IPython's various namespaces
968 965 #-------------------------------------------------------------------------
969 966 default_user_namespaces = True
970 967
971 968 def init_create_namespaces(self, user_module=None, user_ns=None):
972 969 # Create the namespace where the user will operate. user_ns is
973 970 # normally the only one used, and it is passed to the exec calls as
974 971 # the locals argument. But we do carry a user_global_ns namespace
975 972 # given as the exec 'globals' argument, This is useful in embedding
976 973 # situations where the ipython shell opens in a context where the
977 974 # distinction between locals and globals is meaningful. For
978 975 # non-embedded contexts, it is just the same object as the user_ns dict.
979 976
980 977 # FIXME. For some strange reason, __builtins__ is showing up at user
981 978 # level as a dict instead of a module. This is a manual fix, but I
982 979 # should really track down where the problem is coming from. Alex
983 980 # Schmolck reported this problem first.
984 981
985 982 # A useful post by Alex Martelli on this topic:
986 983 # Re: inconsistent value from __builtins__
987 984 # Von: Alex Martelli <aleaxit@yahoo.com>
988 985 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
989 986 # Gruppen: comp.lang.python
990 987
991 988 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
992 989 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
993 990 # > <type 'dict'>
994 991 # > >>> print type(__builtins__)
995 992 # > <type 'module'>
996 993 # > Is this difference in return value intentional?
997 994
998 995 # Well, it's documented that '__builtins__' can be either a dictionary
999 996 # or a module, and it's been that way for a long time. Whether it's
1000 997 # intentional (or sensible), I don't know. In any case, the idea is
1001 998 # that if you need to access the built-in namespace directly, you
1002 999 # should start with "import __builtin__" (note, no 's') which will
1003 1000 # definitely give you a module. Yeah, it's somewhat confusing:-(.
1004 1001
1005 1002 # These routines return a properly built module and dict as needed by
1006 1003 # the rest of the code, and can also be used by extension writers to
1007 1004 # generate properly initialized namespaces.
1008 1005 if (user_ns is not None) or (user_module is not None):
1009 1006 self.default_user_namespaces = False
1010 1007 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
1011 1008
1012 1009 # A record of hidden variables we have added to the user namespace, so
1013 1010 # we can list later only variables defined in actual interactive use.
1014 1011 self.user_ns_hidden = {}
1015 1012
1016 1013 # Now that FakeModule produces a real module, we've run into a nasty
1017 1014 # problem: after script execution (via %run), the module where the user
1018 1015 # code ran is deleted. Now that this object is a true module (needed
1019 1016 # so doctest and other tools work correctly), the Python module
1020 1017 # teardown mechanism runs over it, and sets to None every variable
1021 1018 # present in that module. Top-level references to objects from the
1022 1019 # script survive, because the user_ns is updated with them. However,
1023 1020 # calling functions defined in the script that use other things from
1024 1021 # the script will fail, because the function's closure had references
1025 1022 # to the original objects, which are now all None. So we must protect
1026 1023 # these modules from deletion by keeping a cache.
1027 1024 #
1028 1025 # To avoid keeping stale modules around (we only need the one from the
1029 1026 # last run), we use a dict keyed with the full path to the script, so
1030 1027 # only the last version of the module is held in the cache. Note,
1031 1028 # however, that we must cache the module *namespace contents* (their
1032 1029 # __dict__). Because if we try to cache the actual modules, old ones
1033 1030 # (uncached) could be destroyed while still holding references (such as
1034 1031 # those held by GUI objects that tend to be long-lived)>
1035 1032 #
1036 1033 # The %reset command will flush this cache. See the cache_main_mod()
1037 1034 # and clear_main_mod_cache() methods for details on use.
1038 1035
1039 1036 # This is the cache used for 'main' namespaces
1040 1037 self._main_mod_cache = {}
1041 1038
1042 1039 # A table holding all the namespaces IPython deals with, so that
1043 1040 # introspection facilities can search easily.
1044 1041 self.ns_table = {'user_global':self.user_module.__dict__,
1045 1042 'user_local':self.user_ns,
1046 1043 'builtin':builtin_mod.__dict__
1047 1044 }
1048 1045
1049 1046 @property
1050 1047 def user_global_ns(self):
1051 1048 return self.user_module.__dict__
1052 1049
1053 1050 def prepare_user_module(self, user_module=None, user_ns=None):
1054 1051 """Prepare the module and namespace in which user code will be run.
1055 1052
1056 1053 When IPython is started normally, both parameters are None: a new module
1057 1054 is created automatically, and its __dict__ used as the namespace.
1058 1055
1059 1056 If only user_module is provided, its __dict__ is used as the namespace.
1060 1057 If only user_ns is provided, a dummy module is created, and user_ns
1061 1058 becomes the global namespace. If both are provided (as they may be
1062 1059 when embedding), user_ns is the local namespace, and user_module
1063 1060 provides the global namespace.
1064 1061
1065 1062 Parameters
1066 1063 ----------
1067 1064 user_module : module, optional
1068 1065 The current user module in which IPython is being run. If None,
1069 1066 a clean module will be created.
1070 1067 user_ns : dict, optional
1071 1068 A namespace in which to run interactive commands.
1072 1069
1073 1070 Returns
1074 1071 -------
1075 1072 A tuple of user_module and user_ns, each properly initialised.
1076 1073 """
1077 1074 if user_module is None and user_ns is not None:
1078 1075 user_ns.setdefault("__name__", "__main__")
1079 1076 user_module = DummyMod()
1080 1077 user_module.__dict__ = user_ns
1081 1078
1082 1079 if user_module is None:
1083 1080 user_module = types.ModuleType("__main__",
1084 1081 doc="Automatically created module for IPython interactive environment")
1085 1082
1086 1083 # We must ensure that __builtin__ (without the final 's') is always
1087 1084 # available and pointing to the __builtin__ *module*. For more details:
1088 1085 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1089 1086 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1090 1087 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1091 1088
1092 1089 if user_ns is None:
1093 1090 user_ns = user_module.__dict__
1094 1091
1095 1092 return user_module, user_ns
1096 1093
1097 1094 def init_sys_modules(self):
1098 1095 # We need to insert into sys.modules something that looks like a
1099 1096 # module but which accesses the IPython namespace, for shelve and
1100 1097 # pickle to work interactively. Normally they rely on getting
1101 1098 # everything out of __main__, but for embedding purposes each IPython
1102 1099 # instance has its own private namespace, so we can't go shoving
1103 1100 # everything into __main__.
1104 1101
1105 1102 # note, however, that we should only do this for non-embedded
1106 1103 # ipythons, which really mimic the __main__.__dict__ with their own
1107 1104 # namespace. Embedded instances, on the other hand, should not do
1108 1105 # this because they need to manage the user local/global namespaces
1109 1106 # only, but they live within a 'normal' __main__ (meaning, they
1110 1107 # shouldn't overtake the execution environment of the script they're
1111 1108 # embedded in).
1112 1109
1113 1110 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1114 1111 main_name = self.user_module.__name__
1115 1112 sys.modules[main_name] = self.user_module
1116 1113
1117 1114 def init_user_ns(self):
1118 1115 """Initialize all user-visible namespaces to their minimum defaults.
1119 1116
1120 1117 Certain history lists are also initialized here, as they effectively
1121 1118 act as user namespaces.
1122 1119
1123 1120 Notes
1124 1121 -----
1125 1122 All data structures here are only filled in, they are NOT reset by this
1126 1123 method. If they were not empty before, data will simply be added to
1127 1124 therm.
1128 1125 """
1129 1126 # This function works in two parts: first we put a few things in
1130 1127 # user_ns, and we sync that contents into user_ns_hidden so that these
1131 1128 # initial variables aren't shown by %who. After the sync, we add the
1132 1129 # rest of what we *do* want the user to see with %who even on a new
1133 1130 # session (probably nothing, so they really only see their own stuff)
1134 1131
1135 1132 # The user dict must *always* have a __builtin__ reference to the
1136 1133 # Python standard __builtin__ namespace, which must be imported.
1137 1134 # This is so that certain operations in prompt evaluation can be
1138 1135 # reliably executed with builtins. Note that we can NOT use
1139 1136 # __builtins__ (note the 's'), because that can either be a dict or a
1140 1137 # module, and can even mutate at runtime, depending on the context
1141 1138 # (Python makes no guarantees on it). In contrast, __builtin__ is
1142 1139 # always a module object, though it must be explicitly imported.
1143 1140
1144 1141 # For more details:
1145 1142 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1146 1143 ns = dict()
1147 1144
1148 1145 # make global variables for user access to the histories
1149 1146 ns['_ih'] = self.history_manager.input_hist_parsed
1150 1147 ns['_oh'] = self.history_manager.output_hist
1151 1148 ns['_dh'] = self.history_manager.dir_hist
1152 1149
1153 1150 ns['_sh'] = shadowns
1154 1151
1155 1152 # user aliases to input and output histories. These shouldn't show up
1156 1153 # in %who, as they can have very large reprs.
1157 1154 ns['In'] = self.history_manager.input_hist_parsed
1158 1155 ns['Out'] = self.history_manager.output_hist
1159 1156
1160 1157 # Store myself as the public api!!!
1161 1158 ns['get_ipython'] = self.get_ipython
1162 1159
1163 1160 ns['exit'] = self.exiter
1164 1161 ns['quit'] = self.exiter
1165 1162
1166 1163 # Sync what we've added so far to user_ns_hidden so these aren't seen
1167 1164 # by %who
1168 1165 self.user_ns_hidden.update(ns)
1169 1166
1170 1167 # Anything put into ns now would show up in %who. Think twice before
1171 1168 # putting anything here, as we really want %who to show the user their
1172 1169 # stuff, not our variables.
1173 1170
1174 1171 # Finally, update the real user's namespace
1175 1172 self.user_ns.update(ns)
1176 1173
1177 1174 @property
1178 1175 def all_ns_refs(self):
1179 1176 """Get a list of references to all the namespace dictionaries in which
1180 1177 IPython might store a user-created object.
1181 1178
1182 1179 Note that this does not include the displayhook, which also caches
1183 1180 objects from the output."""
1184 1181 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1185 1182 [m.__dict__ for m in self._main_mod_cache.values()]
1186 1183
1187 1184 def reset(self, new_session=True):
1188 1185 """Clear all internal namespaces, and attempt to release references to
1189 1186 user objects.
1190 1187
1191 1188 If new_session is True, a new history session will be opened.
1192 1189 """
1193 1190 # Clear histories
1194 1191 self.history_manager.reset(new_session)
1195 1192 # Reset counter used to index all histories
1196 1193 if new_session:
1197 1194 self.execution_count = 1
1198 1195
1199 1196 # Flush cached output items
1200 1197 if self.displayhook.do_full_cache:
1201 1198 self.displayhook.flush()
1202 1199
1203 1200 # The main execution namespaces must be cleared very carefully,
1204 1201 # skipping the deletion of the builtin-related keys, because doing so
1205 1202 # would cause errors in many object's __del__ methods.
1206 1203 if self.user_ns is not self.user_global_ns:
1207 1204 self.user_ns.clear()
1208 1205 ns = self.user_global_ns
1209 1206 drop_keys = set(ns.keys())
1210 1207 drop_keys.discard('__builtin__')
1211 1208 drop_keys.discard('__builtins__')
1212 1209 drop_keys.discard('__name__')
1213 1210 for k in drop_keys:
1214 1211 del ns[k]
1215 1212
1216 1213 self.user_ns_hidden.clear()
1217 1214
1218 1215 # Restore the user namespaces to minimal usability
1219 1216 self.init_user_ns()
1220 1217
1221 1218 # Restore the default and user aliases
1222 1219 self.alias_manager.clear_aliases()
1223 1220 self.alias_manager.init_aliases()
1224 1221
1225 1222 # Flush the private list of module references kept for script
1226 1223 # execution protection
1227 1224 self.clear_main_mod_cache()
1228 1225
1229 1226 def del_var(self, varname, by_name=False):
1230 1227 """Delete a variable from the various namespaces, so that, as
1231 1228 far as possible, we're not keeping any hidden references to it.
1232 1229
1233 1230 Parameters
1234 1231 ----------
1235 1232 varname : str
1236 1233 The name of the variable to delete.
1237 1234 by_name : bool
1238 1235 If True, delete variables with the given name in each
1239 1236 namespace. If False (default), find the variable in the user
1240 1237 namespace, and delete references to it.
1241 1238 """
1242 1239 if varname in ('__builtin__', '__builtins__'):
1243 1240 raise ValueError("Refusing to delete %s" % varname)
1244 1241
1245 1242 ns_refs = self.all_ns_refs
1246 1243
1247 1244 if by_name: # Delete by name
1248 1245 for ns in ns_refs:
1249 1246 try:
1250 1247 del ns[varname]
1251 1248 except KeyError:
1252 1249 pass
1253 1250 else: # Delete by object
1254 1251 try:
1255 1252 obj = self.user_ns[varname]
1256 1253 except KeyError:
1257 1254 raise NameError("name '%s' is not defined" % varname)
1258 1255 # Also check in output history
1259 1256 ns_refs.append(self.history_manager.output_hist)
1260 1257 for ns in ns_refs:
1261 1258 to_delete = [n for n, o in iteritems(ns) if o is obj]
1262 1259 for name in to_delete:
1263 1260 del ns[name]
1264 1261
1265 1262 # displayhook keeps extra references, but not in a dictionary
1266 1263 for name in ('_', '__', '___'):
1267 1264 if getattr(self.displayhook, name) is obj:
1268 1265 setattr(self.displayhook, name, None)
1269 1266
1270 1267 def reset_selective(self, regex=None):
1271 1268 """Clear selective variables from internal namespaces based on a
1272 1269 specified regular expression.
1273 1270
1274 1271 Parameters
1275 1272 ----------
1276 1273 regex : string or compiled pattern, optional
1277 1274 A regular expression pattern that will be used in searching
1278 1275 variable names in the users namespaces.
1279 1276 """
1280 1277 if regex is not None:
1281 1278 try:
1282 1279 m = re.compile(regex)
1283 1280 except TypeError:
1284 1281 raise TypeError('regex must be a string or compiled pattern')
1285 1282 # Search for keys in each namespace that match the given regex
1286 1283 # If a match is found, delete the key/value pair.
1287 1284 for ns in self.all_ns_refs:
1288 1285 for var in ns:
1289 1286 if m.search(var):
1290 1287 del ns[var]
1291 1288
1292 1289 def push(self, variables, interactive=True):
1293 1290 """Inject a group of variables into the IPython user namespace.
1294 1291
1295 1292 Parameters
1296 1293 ----------
1297 1294 variables : dict, str or list/tuple of str
1298 1295 The variables to inject into the user's namespace. If a dict, a
1299 1296 simple update is done. If a str, the string is assumed to have
1300 1297 variable names separated by spaces. A list/tuple of str can also
1301 1298 be used to give the variable names. If just the variable names are
1302 1299 give (list/tuple/str) then the variable values looked up in the
1303 1300 callers frame.
1304 1301 interactive : bool
1305 1302 If True (default), the variables will be listed with the ``who``
1306 1303 magic.
1307 1304 """
1308 1305 vdict = None
1309 1306
1310 1307 # We need a dict of name/value pairs to do namespace updates.
1311 1308 if isinstance(variables, dict):
1312 1309 vdict = variables
1313 1310 elif isinstance(variables, string_types+(list, tuple)):
1314 1311 if isinstance(variables, string_types):
1315 1312 vlist = variables.split()
1316 1313 else:
1317 1314 vlist = variables
1318 1315 vdict = {}
1319 1316 cf = sys._getframe(1)
1320 1317 for name in vlist:
1321 1318 try:
1322 1319 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1323 1320 except:
1324 1321 print('Could not get variable %s from %s' %
1325 1322 (name,cf.f_code.co_name))
1326 1323 else:
1327 1324 raise ValueError('variables must be a dict/str/list/tuple')
1328 1325
1329 1326 # Propagate variables to user namespace
1330 1327 self.user_ns.update(vdict)
1331 1328
1332 1329 # And configure interactive visibility
1333 1330 user_ns_hidden = self.user_ns_hidden
1334 1331 if interactive:
1335 1332 for name in vdict:
1336 1333 user_ns_hidden.pop(name, None)
1337 1334 else:
1338 1335 user_ns_hidden.update(vdict)
1339 1336
1340 1337 def drop_by_id(self, variables):
1341 1338 """Remove a dict of variables from the user namespace, if they are the
1342 1339 same as the values in the dictionary.
1343 1340
1344 1341 This is intended for use by extensions: variables that they've added can
1345 1342 be taken back out if they are unloaded, without removing any that the
1346 1343 user has overwritten.
1347 1344
1348 1345 Parameters
1349 1346 ----------
1350 1347 variables : dict
1351 1348 A dictionary mapping object names (as strings) to the objects.
1352 1349 """
1353 1350 for name, obj in iteritems(variables):
1354 1351 if name in self.user_ns and self.user_ns[name] is obj:
1355 1352 del self.user_ns[name]
1356 1353 self.user_ns_hidden.pop(name, None)
1357 1354
1358 1355 #-------------------------------------------------------------------------
1359 1356 # Things related to object introspection
1360 1357 #-------------------------------------------------------------------------
1361 1358
1362 1359 def _ofind(self, oname, namespaces=None):
1363 1360 """Find an object in the available namespaces.
1364 1361
1365 1362 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1366 1363
1367 1364 Has special code to detect magic functions.
1368 1365 """
1369 1366 oname = oname.strip()
1370 1367 #print '1- oname: <%r>' % oname # dbg
1371 1368 if not oname.startswith(ESC_MAGIC) and \
1372 1369 not oname.startswith(ESC_MAGIC2) and \
1373 1370 not py3compat.isidentifier(oname, dotted=True):
1374 1371 return dict(found=False)
1375 1372
1376 1373 if namespaces is None:
1377 1374 # Namespaces to search in:
1378 1375 # Put them in a list. The order is important so that we
1379 1376 # find things in the same order that Python finds them.
1380 1377 namespaces = [ ('Interactive', self.user_ns),
1381 1378 ('Interactive (global)', self.user_global_ns),
1382 1379 ('Python builtin', builtin_mod.__dict__),
1383 1380 ]
1384 1381
1385 1382 # initialize results to 'null'
1386 1383 found = False; obj = None; ospace = None;
1387 1384 ismagic = False; isalias = False; parent = None
1388 1385
1389 1386 # We need to special-case 'print', which as of python2.6 registers as a
1390 1387 # function but should only be treated as one if print_function was
1391 1388 # loaded with a future import. In this case, just bail.
1392 1389 if (oname == 'print' and not py3compat.PY3 and not \
1393 1390 (self.compile.compiler_flags & __future__.CO_FUTURE_PRINT_FUNCTION)):
1394 1391 return {'found':found, 'obj':obj, 'namespace':ospace,
1395 1392 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1396 1393
1397 1394 # Look for the given name by splitting it in parts. If the head is
1398 1395 # found, then we look for all the remaining parts as members, and only
1399 1396 # declare success if we can find them all.
1400 1397 oname_parts = oname.split('.')
1401 1398 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1402 1399 for nsname,ns in namespaces:
1403 1400 try:
1404 1401 obj = ns[oname_head]
1405 1402 except KeyError:
1406 1403 continue
1407 1404 else:
1408 1405 #print 'oname_rest:', oname_rest # dbg
1409 1406 for idx, part in enumerate(oname_rest):
1410 1407 try:
1411 1408 parent = obj
1412 1409 # The last part is looked up in a special way to avoid
1413 1410 # descriptor invocation as it may raise or have side
1414 1411 # effects.
1415 1412 if idx == len(oname_rest) - 1:
1416 1413 obj = self._getattr_property(obj, part)
1417 1414 else:
1418 1415 obj = getattr(obj, part)
1419 1416 except:
1420 1417 # Blanket except b/c some badly implemented objects
1421 1418 # allow __getattr__ to raise exceptions other than
1422 1419 # AttributeError, which then crashes IPython.
1423 1420 break
1424 1421 else:
1425 1422 # If we finish the for loop (no break), we got all members
1426 1423 found = True
1427 1424 ospace = nsname
1428 1425 break # namespace loop
1429 1426
1430 1427 # Try to see if it's magic
1431 1428 if not found:
1432 1429 obj = None
1433 1430 if oname.startswith(ESC_MAGIC2):
1434 1431 oname = oname.lstrip(ESC_MAGIC2)
1435 1432 obj = self.find_cell_magic(oname)
1436 1433 elif oname.startswith(ESC_MAGIC):
1437 1434 oname = oname.lstrip(ESC_MAGIC)
1438 1435 obj = self.find_line_magic(oname)
1439 1436 else:
1440 1437 # search without prefix, so run? will find %run?
1441 1438 obj = self.find_line_magic(oname)
1442 1439 if obj is None:
1443 1440 obj = self.find_cell_magic(oname)
1444 1441 if obj is not None:
1445 1442 found = True
1446 1443 ospace = 'IPython internal'
1447 1444 ismagic = True
1448 1445 isalias = isinstance(obj, Alias)
1449 1446
1450 1447 # Last try: special-case some literals like '', [], {}, etc:
1451 1448 if not found and oname_head in ["''",'""','[]','{}','()']:
1452 1449 obj = eval(oname_head)
1453 1450 found = True
1454 1451 ospace = 'Interactive'
1455 1452
1456 1453 return {'found':found, 'obj':obj, 'namespace':ospace,
1457 1454 'ismagic':ismagic, 'isalias':isalias, 'parent':parent}
1458 1455
1459 1456 @staticmethod
1460 1457 def _getattr_property(obj, attrname):
1461 1458 """Property-aware getattr to use in object finding.
1462 1459
1463 1460 If attrname represents a property, return it unevaluated (in case it has
1464 1461 side effects or raises an error.
1465 1462
1466 1463 """
1467 1464 if not isinstance(obj, type):
1468 1465 try:
1469 1466 # `getattr(type(obj), attrname)` is not guaranteed to return
1470 1467 # `obj`, but does so for property:
1471 1468 #
1472 1469 # property.__get__(self, None, cls) -> self
1473 1470 #
1474 1471 # The universal alternative is to traverse the mro manually
1475 1472 # searching for attrname in class dicts.
1476 1473 attr = getattr(type(obj), attrname)
1477 1474 except AttributeError:
1478 1475 pass
1479 1476 else:
1480 1477 # This relies on the fact that data descriptors (with both
1481 1478 # __get__ & __set__ magic methods) take precedence over
1482 1479 # instance-level attributes:
1483 1480 #
1484 1481 # class A(object):
1485 1482 # @property
1486 1483 # def foobar(self): return 123
1487 1484 # a = A()
1488 1485 # a.__dict__['foobar'] = 345
1489 1486 # a.foobar # == 123
1490 1487 #
1491 1488 # So, a property may be returned right away.
1492 1489 if isinstance(attr, property):
1493 1490 return attr
1494 1491
1495 1492 # Nothing helped, fall back.
1496 1493 return getattr(obj, attrname)
1497 1494
1498 1495 def _object_find(self, oname, namespaces=None):
1499 1496 """Find an object and return a struct with info about it."""
1500 1497 return Struct(self._ofind(oname, namespaces))
1501 1498
1502 1499 def _inspect(self, meth, oname, namespaces=None, **kw):
1503 1500 """Generic interface to the inspector system.
1504 1501
1505 1502 This function is meant to be called by pdef, pdoc & friends.
1506 1503 """
1507 1504 info = self._object_find(oname, namespaces)
1508 1505 docformat = sphinxify if self.sphinxify_docstring else None
1509 1506 if info.found:
1510 1507 pmethod = getattr(self.inspector, meth)
1511 1508 # TODO: only apply format_screen to the plain/text repr of the mime
1512 1509 # bundle.
1513 1510 formatter = format_screen if info.ismagic else docformat
1514 1511 if meth == 'pdoc':
1515 1512 pmethod(info.obj, oname, formatter)
1516 1513 elif meth == 'pinfo':
1517 1514 pmethod(info.obj, oname, formatter, info,
1518 1515 enable_html_pager=self.enable_html_pager, **kw)
1519 1516 else:
1520 1517 pmethod(info.obj, oname)
1521 1518 else:
1522 1519 print('Object `%s` not found.' % oname)
1523 1520 return 'not found' # so callers can take other action
1524 1521
1525 1522 def object_inspect(self, oname, detail_level=0):
1526 1523 """Get object info about oname"""
1527 1524 with self.builtin_trap:
1528 1525 info = self._object_find(oname)
1529 1526 if info.found:
1530 1527 return self.inspector.info(info.obj, oname, info=info,
1531 1528 detail_level=detail_level
1532 1529 )
1533 1530 else:
1534 1531 return oinspect.object_info(name=oname, found=False)
1535 1532
1536 1533 def object_inspect_text(self, oname, detail_level=0):
1537 1534 """Get object info as formatted text"""
1538 1535 return self.object_inspect_mime(oname, detail_level)['text/plain']
1539 1536
1540 1537 def object_inspect_mime(self, oname, detail_level=0):
1541 1538 """Get object info as a mimebundle of formatted representations.
1542 1539
1543 1540 A mimebundle is a dictionary, keyed by mime-type.
1544 1541 It must always have the key `'text/plain'`.
1545 1542 """
1546 1543 with self.builtin_trap:
1547 1544 info = self._object_find(oname)
1548 1545 if info.found:
1549 1546 return self.inspector._get_info(info.obj, oname, info=info,
1550 1547 detail_level=detail_level
1551 1548 )
1552 1549 else:
1553 1550 raise KeyError(oname)
1554 1551
1555 1552 #-------------------------------------------------------------------------
1556 1553 # Things related to history management
1557 1554 #-------------------------------------------------------------------------
1558 1555
1559 1556 def init_history(self):
1560 1557 """Sets up the command history, and starts regular autosaves."""
1561 1558 self.history_manager = HistoryManager(shell=self, parent=self)
1562 1559 self.configurables.append(self.history_manager)
1563 1560
1564 1561 #-------------------------------------------------------------------------
1565 1562 # Things related to exception handling and tracebacks (not debugging)
1566 1563 #-------------------------------------------------------------------------
1567 1564
1568 1565 debugger_cls = Pdb
1569 1566
1570 1567 def init_traceback_handlers(self, custom_exceptions):
1571 1568 # Syntax error handler.
1572 1569 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
1573 1570
1574 1571 # The interactive one is initialized with an offset, meaning we always
1575 1572 # want to remove the topmost item in the traceback, which is our own
1576 1573 # internal code. Valid modes: ['Plain','Context','Verbose']
1577 1574 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1578 1575 color_scheme='NoColor',
1579 1576 tb_offset = 1,
1580 1577 check_cache=check_linecache_ipython,
1581 1578 debugger_cls=self.debugger_cls, parent=self)
1582 1579
1583 1580 # The instance will store a pointer to the system-wide exception hook,
1584 1581 # so that runtime code (such as magics) can access it. This is because
1585 1582 # during the read-eval loop, it may get temporarily overwritten.
1586 1583 self.sys_excepthook = sys.excepthook
1587 1584
1588 1585 # and add any custom exception handlers the user may have specified
1589 1586 self.set_custom_exc(*custom_exceptions)
1590 1587
1591 1588 # Set the exception mode
1592 1589 self.InteractiveTB.set_mode(mode=self.xmode)
1593 1590
1594 1591 def set_custom_exc(self, exc_tuple, handler):
1595 1592 """set_custom_exc(exc_tuple, handler)
1596 1593
1597 1594 Set a custom exception handler, which will be called if any of the
1598 1595 exceptions in exc_tuple occur in the mainloop (specifically, in the
1599 1596 run_code() method).
1600 1597
1601 1598 Parameters
1602 1599 ----------
1603 1600
1604 1601 exc_tuple : tuple of exception classes
1605 1602 A *tuple* of exception classes, for which to call the defined
1606 1603 handler. It is very important that you use a tuple, and NOT A
1607 1604 LIST here, because of the way Python's except statement works. If
1608 1605 you only want to trap a single exception, use a singleton tuple::
1609 1606
1610 1607 exc_tuple == (MyCustomException,)
1611 1608
1612 1609 handler : callable
1613 1610 handler must have the following signature::
1614 1611
1615 1612 def my_handler(self, etype, value, tb, tb_offset=None):
1616 1613 ...
1617 1614 return structured_traceback
1618 1615
1619 1616 Your handler must return a structured traceback (a list of strings),
1620 1617 or None.
1621 1618
1622 1619 This will be made into an instance method (via types.MethodType)
1623 1620 of IPython itself, and it will be called if any of the exceptions
1624 1621 listed in the exc_tuple are caught. If the handler is None, an
1625 1622 internal basic one is used, which just prints basic info.
1626 1623
1627 1624 To protect IPython from crashes, if your handler ever raises an
1628 1625 exception or returns an invalid result, it will be immediately
1629 1626 disabled.
1630 1627
1631 1628 WARNING: by putting in your own exception handler into IPython's main
1632 1629 execution loop, you run a very good chance of nasty crashes. This
1633 1630 facility should only be used if you really know what you are doing."""
1634 1631
1635 1632 assert type(exc_tuple)==type(()) , \
1636 1633 "The custom exceptions must be given AS A TUPLE."
1637 1634
1638 1635 def dummy_handler(self, etype, value, tb, tb_offset=None):
1639 1636 print('*** Simple custom exception handler ***')
1640 1637 print('Exception type :',etype)
1641 1638 print('Exception value:',value)
1642 1639 print('Traceback :',tb)
1643 1640 #print 'Source code :','\n'.join(self.buffer)
1644 1641
1645 1642 def validate_stb(stb):
1646 1643 """validate structured traceback return type
1647 1644
1648 1645 return type of CustomTB *should* be a list of strings, but allow
1649 1646 single strings or None, which are harmless.
1650 1647
1651 1648 This function will *always* return a list of strings,
1652 1649 and will raise a TypeError if stb is inappropriate.
1653 1650 """
1654 1651 msg = "CustomTB must return list of strings, not %r" % stb
1655 1652 if stb is None:
1656 1653 return []
1657 1654 elif isinstance(stb, string_types):
1658 1655 return [stb]
1659 1656 elif not isinstance(stb, list):
1660 1657 raise TypeError(msg)
1661 1658 # it's a list
1662 1659 for line in stb:
1663 1660 # check every element
1664 1661 if not isinstance(line, string_types):
1665 1662 raise TypeError(msg)
1666 1663 return stb
1667 1664
1668 1665 if handler is None:
1669 1666 wrapped = dummy_handler
1670 1667 else:
1671 1668 def wrapped(self,etype,value,tb,tb_offset=None):
1672 1669 """wrap CustomTB handler, to protect IPython from user code
1673 1670
1674 1671 This makes it harder (but not impossible) for custom exception
1675 1672 handlers to crash IPython.
1676 1673 """
1677 1674 try:
1678 1675 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1679 1676 return validate_stb(stb)
1680 1677 except:
1681 1678 # clear custom handler immediately
1682 1679 self.set_custom_exc((), None)
1683 1680 print("Custom TB Handler failed, unregistering", file=sys.stderr)
1684 1681 # show the exception in handler first
1685 1682 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1686 1683 print(self.InteractiveTB.stb2text(stb))
1687 1684 print("The original exception:")
1688 1685 stb = self.InteractiveTB.structured_traceback(
1689 1686 (etype,value,tb), tb_offset=tb_offset
1690 1687 )
1691 1688 return stb
1692 1689
1693 1690 self.CustomTB = types.MethodType(wrapped,self)
1694 1691 self.custom_exceptions = exc_tuple
1695 1692
1696 1693 def excepthook(self, etype, value, tb):
1697 1694 """One more defense for GUI apps that call sys.excepthook.
1698 1695
1699 1696 GUI frameworks like wxPython trap exceptions and call
1700 1697 sys.excepthook themselves. I guess this is a feature that
1701 1698 enables them to keep running after exceptions that would
1702 1699 otherwise kill their mainloop. This is a bother for IPython
1703 1700 which excepts to catch all of the program exceptions with a try:
1704 1701 except: statement.
1705 1702
1706 1703 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1707 1704 any app directly invokes sys.excepthook, it will look to the user like
1708 1705 IPython crashed. In order to work around this, we can disable the
1709 1706 CrashHandler and replace it with this excepthook instead, which prints a
1710 1707 regular traceback using our InteractiveTB. In this fashion, apps which
1711 1708 call sys.excepthook will generate a regular-looking exception from
1712 1709 IPython, and the CrashHandler will only be triggered by real IPython
1713 1710 crashes.
1714 1711
1715 1712 This hook should be used sparingly, only in places which are not likely
1716 1713 to be true IPython errors.
1717 1714 """
1718 1715 self.showtraceback((etype, value, tb), tb_offset=0)
1719 1716
1720 1717 def _get_exc_info(self, exc_tuple=None):
1721 1718 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1722 1719
1723 1720 Ensures sys.last_type,value,traceback hold the exc_info we found,
1724 1721 from whichever source.
1725 1722
1726 1723 raises ValueError if none of these contain any information
1727 1724 """
1728 1725 if exc_tuple is None:
1729 1726 etype, value, tb = sys.exc_info()
1730 1727 else:
1731 1728 etype, value, tb = exc_tuple
1732 1729
1733 1730 if etype is None:
1734 1731 if hasattr(sys, 'last_type'):
1735 1732 etype, value, tb = sys.last_type, sys.last_value, \
1736 1733 sys.last_traceback
1737 1734
1738 1735 if etype is None:
1739 1736 raise ValueError("No exception to find")
1740 1737
1741 1738 # Now store the exception info in sys.last_type etc.
1742 1739 # WARNING: these variables are somewhat deprecated and not
1743 1740 # necessarily safe to use in a threaded environment, but tools
1744 1741 # like pdb depend on their existence, so let's set them. If we
1745 1742 # find problems in the field, we'll need to revisit their use.
1746 1743 sys.last_type = etype
1747 1744 sys.last_value = value
1748 1745 sys.last_traceback = tb
1749 1746
1750 1747 return etype, value, tb
1751 1748
1752 1749 def show_usage_error(self, exc):
1753 1750 """Show a short message for UsageErrors
1754 1751
1755 1752 These are special exceptions that shouldn't show a traceback.
1756 1753 """
1757 1754 print("UsageError: %s" % exc, file=sys.stderr)
1758 1755
1759 1756 def get_exception_only(self, exc_tuple=None):
1760 1757 """
1761 1758 Return as a string (ending with a newline) the exception that
1762 1759 just occurred, without any traceback.
1763 1760 """
1764 1761 etype, value, tb = self._get_exc_info(exc_tuple)
1765 1762 msg = traceback.format_exception_only(etype, value)
1766 1763 return ''.join(msg)
1767 1764
1768 1765 def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
1769 1766 exception_only=False):
1770 1767 """Display the exception that just occurred.
1771 1768
1772 1769 If nothing is known about the exception, this is the method which
1773 1770 should be used throughout the code for presenting user tracebacks,
1774 1771 rather than directly invoking the InteractiveTB object.
1775 1772
1776 1773 A specific showsyntaxerror() also exists, but this method can take
1777 1774 care of calling it if needed, so unless you are explicitly catching a
1778 1775 SyntaxError exception, don't try to analyze the stack manually and
1779 1776 simply call this method."""
1780 1777
1781 1778 try:
1782 1779 try:
1783 1780 etype, value, tb = self._get_exc_info(exc_tuple)
1784 1781 except ValueError:
1785 1782 print('No traceback available to show.', file=sys.stderr)
1786 1783 return
1787 1784
1788 1785 if issubclass(etype, SyntaxError):
1789 1786 # Though this won't be called by syntax errors in the input
1790 1787 # line, there may be SyntaxError cases with imported code.
1791 1788 self.showsyntaxerror(filename)
1792 1789 elif etype is UsageError:
1793 1790 self.show_usage_error(value)
1794 1791 else:
1795 1792 if exception_only:
1796 1793 stb = ['An exception has occurred, use %tb to see '
1797 1794 'the full traceback.\n']
1798 1795 stb.extend(self.InteractiveTB.get_exception_only(etype,
1799 1796 value))
1800 1797 else:
1801 1798 try:
1802 1799 # Exception classes can customise their traceback - we
1803 1800 # use this in IPython.parallel for exceptions occurring
1804 1801 # in the engines. This should return a list of strings.
1805 1802 stb = value._render_traceback_()
1806 1803 except Exception:
1807 1804 stb = self.InteractiveTB.structured_traceback(etype,
1808 1805 value, tb, tb_offset=tb_offset)
1809 1806
1810 1807 self._showtraceback(etype, value, stb)
1811 1808 if self.call_pdb:
1812 1809 # drop into debugger
1813 1810 self.debugger(force=True)
1814 1811 return
1815 1812
1816 1813 # Actually show the traceback
1817 1814 self._showtraceback(etype, value, stb)
1818 1815
1819 1816 except KeyboardInterrupt:
1820 1817 print('\n' + self.get_exception_only(), file=sys.stderr)
1821 1818
1822 1819 def _showtraceback(self, etype, evalue, stb):
1823 1820 """Actually show a traceback.
1824 1821
1825 1822 Subclasses may override this method to put the traceback on a different
1826 1823 place, like a side channel.
1827 1824 """
1828 1825 print(self.InteractiveTB.stb2text(stb))
1829 1826
1830 1827 def showsyntaxerror(self, filename=None):
1831 1828 """Display the syntax error that just occurred.
1832 1829
1833 1830 This doesn't display a stack trace because there isn't one.
1834 1831
1835 1832 If a filename is given, it is stuffed in the exception instead
1836 1833 of what was there before (because Python's parser always uses
1837 1834 "<string>" when reading from a string).
1838 1835 """
1839 1836 etype, value, last_traceback = self._get_exc_info()
1840 1837
1841 1838 if filename and issubclass(etype, SyntaxError):
1842 1839 try:
1843 1840 value.filename = filename
1844 1841 except:
1845 1842 # Not the format we expect; leave it alone
1846 1843 pass
1847 1844
1848 1845 stb = self.SyntaxTB.structured_traceback(etype, value, [])
1849 1846 self._showtraceback(etype, value, stb)
1850 1847
1851 1848 # This is overridden in TerminalInteractiveShell to show a message about
1852 1849 # the %paste magic.
1853 1850 def showindentationerror(self):
1854 1851 """Called by run_cell when there's an IndentationError in code entered
1855 1852 at the prompt.
1856 1853
1857 1854 This is overridden in TerminalInteractiveShell to show a message about
1858 1855 the %paste magic."""
1859 1856 self.showsyntaxerror()
1860 1857
1861 1858 #-------------------------------------------------------------------------
1862 1859 # Things related to readline
1863 1860 #-------------------------------------------------------------------------
1864 1861
1865 1862 def init_readline(self):
1866 1863 """DEPRECATED
1867 1864
1868 1865 Moved to terminal subclass, here only to simplify the init logic."""
1869 1866 # Set a number of methods that depend on readline to be no-op
1870 1867 warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
1871 1868 DeprecationWarning, stacklevel=2)
1872 1869 self.set_custom_completer = no_op
1873 1870
1874 1871 @skip_doctest
1875 1872 def set_next_input(self, s, replace=False):
1876 1873 """ Sets the 'default' input string for the next command line.
1877 1874
1878 1875 Example::
1879 1876
1880 1877 In [1]: _ip.set_next_input("Hello Word")
1881 1878 In [2]: Hello Word_ # cursor is here
1882 1879 """
1883 1880 self.rl_next_input = py3compat.cast_bytes_py2(s)
1884 1881
1885 1882 def _indent_current_str(self):
1886 1883 """return the current level of indentation as a string"""
1887 1884 return self.input_splitter.indent_spaces * ' '
1888 1885
1889 1886 #-------------------------------------------------------------------------
1890 1887 # Things related to text completion
1891 1888 #-------------------------------------------------------------------------
1892 1889
1893 1890 def init_completer(self):
1894 1891 """Initialize the completion machinery.
1895 1892
1896 1893 This creates completion machinery that can be used by client code,
1897 1894 either interactively in-process (typically triggered by the readline
1898 1895 library), programmatically (such as in test suites) or out-of-process
1899 1896 (typically over the network by remote frontends).
1900 1897 """
1901 1898 from IPython.core.completer import IPCompleter
1902 1899 from IPython.core.completerlib import (module_completer,
1903 1900 magic_run_completer, cd_completer, reset_completer)
1904 1901
1905 1902 self.Completer = IPCompleter(shell=self,
1906 1903 namespace=self.user_ns,
1907 1904 global_namespace=self.user_global_ns,
1908 1905 parent=self,
1909 1906 )
1910 1907 self.configurables.append(self.Completer)
1911 1908
1912 1909 # Add custom completers to the basic ones built into IPCompleter
1913 1910 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
1914 1911 self.strdispatchers['complete_command'] = sdisp
1915 1912 self.Completer.custom_completers = sdisp
1916 1913
1917 1914 self.set_hook('complete_command', module_completer, str_key = 'import')
1918 1915 self.set_hook('complete_command', module_completer, str_key = 'from')
1919 1916 self.set_hook('complete_command', module_completer, str_key = '%aimport')
1920 1917 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
1921 1918 self.set_hook('complete_command', cd_completer, str_key = '%cd')
1922 1919 self.set_hook('complete_command', reset_completer, str_key = '%reset')
1923 1920
1924 1921
1925 1922 @skip_doctest_py2
1926 1923 def complete(self, text, line=None, cursor_pos=None):
1927 1924 """Return the completed text and a list of completions.
1928 1925
1929 1926 Parameters
1930 1927 ----------
1931 1928
1932 1929 text : string
1933 1930 A string of text to be completed on. It can be given as empty and
1934 1931 instead a line/position pair are given. In this case, the
1935 1932 completer itself will split the line like readline does.
1936 1933
1937 1934 line : string, optional
1938 1935 The complete line that text is part of.
1939 1936
1940 1937 cursor_pos : int, optional
1941 1938 The position of the cursor on the input line.
1942 1939
1943 1940 Returns
1944 1941 -------
1945 1942 text : string
1946 1943 The actual text that was completed.
1947 1944
1948 1945 matches : list
1949 1946 A sorted list with all possible completions.
1950 1947
1951 1948 The optional arguments allow the completion to take more context into
1952 1949 account, and are part of the low-level completion API.
1953 1950
1954 1951 This is a wrapper around the completion mechanism, similar to what
1955 1952 readline does at the command line when the TAB key is hit. By
1956 1953 exposing it as a method, it can be used by other non-readline
1957 1954 environments (such as GUIs) for text completion.
1958 1955
1959 1956 Simple usage example:
1960 1957
1961 1958 In [1]: x = 'hello'
1962 1959
1963 1960 In [2]: _ip.complete('x.l')
1964 1961 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
1965 1962 """
1966 1963
1967 1964 # Inject names into __builtin__ so we can complete on the added names.
1968 1965 with self.builtin_trap:
1969 1966 return self.Completer.complete(text, line, cursor_pos)
1970 1967
1971 1968 def set_custom_completer(self, completer, pos=0):
1972 1969 """Adds a new custom completer function.
1973 1970
1974 1971 The position argument (defaults to 0) is the index in the completers
1975 1972 list where you want the completer to be inserted."""
1976 1973
1977 1974 newcomp = types.MethodType(completer,self.Completer)
1978 1975 self.Completer.matchers.insert(pos,newcomp)
1979 1976
1980 1977 def set_completer_frame(self, frame=None):
1981 1978 """Set the frame of the completer."""
1982 1979 if frame:
1983 1980 self.Completer.namespace = frame.f_locals
1984 1981 self.Completer.global_namespace = frame.f_globals
1985 1982 else:
1986 1983 self.Completer.namespace = self.user_ns
1987 1984 self.Completer.global_namespace = self.user_global_ns
1988 1985
1989 1986 #-------------------------------------------------------------------------
1990 1987 # Things related to magics
1991 1988 #-------------------------------------------------------------------------
1992 1989
1993 1990 def init_magics(self):
1994 1991 from IPython.core import magics as m
1995 1992 self.magics_manager = magic.MagicsManager(shell=self,
1996 1993 parent=self,
1997 1994 user_magics=m.UserMagics(self))
1998 1995 self.configurables.append(self.magics_manager)
1999 1996
2000 1997 # Expose as public API from the magics manager
2001 1998 self.register_magics = self.magics_manager.register
2002 1999
2003 2000 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2004 2001 m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
2005 2002 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2006 2003 m.NamespaceMagics, m.OSMagics, m.PylabMagics, m.ScriptMagics,
2007 2004 )
2008 2005
2009 2006 # Register Magic Aliases
2010 2007 mman = self.magics_manager
2011 2008 # FIXME: magic aliases should be defined by the Magics classes
2012 2009 # or in MagicsManager, not here
2013 2010 mman.register_alias('ed', 'edit')
2014 2011 mman.register_alias('hist', 'history')
2015 2012 mman.register_alias('rep', 'recall')
2016 2013 mman.register_alias('SVG', 'svg', 'cell')
2017 2014 mman.register_alias('HTML', 'html', 'cell')
2018 2015 mman.register_alias('file', 'writefile', 'cell')
2019 2016
2020 2017 # FIXME: Move the color initialization to the DisplayHook, which
2021 2018 # should be split into a prompt manager and displayhook. We probably
2022 2019 # even need a centralize colors management object.
2023 2020 self.magic('colors %s' % self.colors)
2024 2021
2025 2022 # Defined here so that it's included in the documentation
2026 2023 @functools.wraps(magic.MagicsManager.register_function)
2027 2024 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2028 2025 self.magics_manager.register_function(func,
2029 2026 magic_kind=magic_kind, magic_name=magic_name)
2030 2027
2031 2028 def run_line_magic(self, magic_name, line):
2032 2029 """Execute the given line magic.
2033 2030
2034 2031 Parameters
2035 2032 ----------
2036 2033 magic_name : str
2037 2034 Name of the desired magic function, without '%' prefix.
2038 2035
2039 2036 line : str
2040 2037 The rest of the input line as a single string.
2041 2038 """
2042 2039 fn = self.find_line_magic(magic_name)
2043 2040 if fn is None:
2044 2041 cm = self.find_cell_magic(magic_name)
2045 2042 etpl = "Line magic function `%%%s` not found%s."
2046 2043 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2047 2044 'did you mean that instead?)' % magic_name )
2048 2045 error(etpl % (magic_name, extra))
2049 2046 else:
2050 2047 # Note: this is the distance in the stack to the user's frame.
2051 2048 # This will need to be updated if the internal calling logic gets
2052 2049 # refactored, or else we'll be expanding the wrong variables.
2053 2050 stack_depth = 2
2054 2051 magic_arg_s = self.var_expand(line, stack_depth)
2055 2052 # Put magic args in a list so we can call with f(*a) syntax
2056 2053 args = [magic_arg_s]
2057 2054 kwargs = {}
2058 2055 # Grab local namespace if we need it:
2059 2056 if getattr(fn, "needs_local_scope", False):
2060 2057 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2061 2058 with self.builtin_trap:
2062 2059 result = fn(*args,**kwargs)
2063 2060 return result
2064 2061
2065 2062 def run_cell_magic(self, magic_name, line, cell):
2066 2063 """Execute the given cell magic.
2067 2064
2068 2065 Parameters
2069 2066 ----------
2070 2067 magic_name : str
2071 2068 Name of the desired magic function, without '%' prefix.
2072 2069
2073 2070 line : str
2074 2071 The rest of the first input line as a single string.
2075 2072
2076 2073 cell : str
2077 2074 The body of the cell as a (possibly multiline) string.
2078 2075 """
2079 2076 fn = self.find_cell_magic(magic_name)
2080 2077 if fn is None:
2081 2078 lm = self.find_line_magic(magic_name)
2082 2079 etpl = "Cell magic `%%{0}` not found{1}."
2083 2080 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2084 2081 'did you mean that instead?)'.format(magic_name))
2085 2082 error(etpl.format(magic_name, extra))
2086 2083 elif cell == '':
2087 2084 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2088 2085 if self.find_line_magic(magic_name) is not None:
2089 2086 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2090 2087 raise UsageError(message)
2091 2088 else:
2092 2089 # Note: this is the distance in the stack to the user's frame.
2093 2090 # This will need to be updated if the internal calling logic gets
2094 2091 # refactored, or else we'll be expanding the wrong variables.
2095 2092 stack_depth = 2
2096 2093 magic_arg_s = self.var_expand(line, stack_depth)
2097 2094 with self.builtin_trap:
2098 2095 result = fn(magic_arg_s, cell)
2099 2096 return result
2100 2097
2101 2098 def find_line_magic(self, magic_name):
2102 2099 """Find and return a line magic by name.
2103 2100
2104 2101 Returns None if the magic isn't found."""
2105 2102 return self.magics_manager.magics['line'].get(magic_name)
2106 2103
2107 2104 def find_cell_magic(self, magic_name):
2108 2105 """Find and return a cell magic by name.
2109 2106
2110 2107 Returns None if the magic isn't found."""
2111 2108 return self.magics_manager.magics['cell'].get(magic_name)
2112 2109
2113 2110 def find_magic(self, magic_name, magic_kind='line'):
2114 2111 """Find and return a magic of the given type by name.
2115 2112
2116 2113 Returns None if the magic isn't found."""
2117 2114 return self.magics_manager.magics[magic_kind].get(magic_name)
2118 2115
2119 2116 def magic(self, arg_s):
2120 2117 """DEPRECATED. Use run_line_magic() instead.
2121 2118
2122 2119 Call a magic function by name.
2123 2120
2124 2121 Input: a string containing the name of the magic function to call and
2125 2122 any additional arguments to be passed to the magic.
2126 2123
2127 2124 magic('name -opt foo bar') is equivalent to typing at the ipython
2128 2125 prompt:
2129 2126
2130 2127 In[1]: %name -opt foo bar
2131 2128
2132 2129 To call a magic without arguments, simply use magic('name').
2133 2130
2134 2131 This provides a proper Python function to call IPython's magics in any
2135 2132 valid Python code you can type at the interpreter, including loops and
2136 2133 compound statements.
2137 2134 """
2138 2135 # TODO: should we issue a loud deprecation warning here?
2139 2136 magic_name, _, magic_arg_s = arg_s.partition(' ')
2140 2137 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2141 2138 return self.run_line_magic(magic_name, magic_arg_s)
2142 2139
2143 2140 #-------------------------------------------------------------------------
2144 2141 # Things related to macros
2145 2142 #-------------------------------------------------------------------------
2146 2143
2147 2144 def define_macro(self, name, themacro):
2148 2145 """Define a new macro
2149 2146
2150 2147 Parameters
2151 2148 ----------
2152 2149 name : str
2153 2150 The name of the macro.
2154 2151 themacro : str or Macro
2155 2152 The action to do upon invoking the macro. If a string, a new
2156 2153 Macro object is created by passing the string to it.
2157 2154 """
2158 2155
2159 2156 from IPython.core import macro
2160 2157
2161 2158 if isinstance(themacro, string_types):
2162 2159 themacro = macro.Macro(themacro)
2163 2160 if not isinstance(themacro, macro.Macro):
2164 2161 raise ValueError('A macro must be a string or a Macro instance.')
2165 2162 self.user_ns[name] = themacro
2166 2163
2167 2164 #-------------------------------------------------------------------------
2168 2165 # Things related to the running of system commands
2169 2166 #-------------------------------------------------------------------------
2170 2167
2171 2168 def system_piped(self, cmd):
2172 2169 """Call the given cmd in a subprocess, piping stdout/err
2173 2170
2174 2171 Parameters
2175 2172 ----------
2176 2173 cmd : str
2177 2174 Command to execute (can not end in '&', as background processes are
2178 2175 not supported. Should not be a command that expects input
2179 2176 other than simple text.
2180 2177 """
2181 2178 if cmd.rstrip().endswith('&'):
2182 2179 # this is *far* from a rigorous test
2183 2180 # We do not support backgrounding processes because we either use
2184 2181 # pexpect or pipes to read from. Users can always just call
2185 2182 # os.system() or use ip.system=ip.system_raw
2186 2183 # if they really want a background process.
2187 2184 raise OSError("Background processes not supported.")
2188 2185
2189 2186 # we explicitly do NOT return the subprocess status code, because
2190 2187 # a non-None value would trigger :func:`sys.displayhook` calls.
2191 2188 # Instead, we store the exit_code in user_ns.
2192 2189 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2193 2190
2194 2191 def system_raw(self, cmd):
2195 2192 """Call the given cmd in a subprocess using os.system on Windows or
2196 2193 subprocess.call using the system shell on other platforms.
2197 2194
2198 2195 Parameters
2199 2196 ----------
2200 2197 cmd : str
2201 2198 Command to execute.
2202 2199 """
2203 2200 cmd = self.var_expand(cmd, depth=1)
2204 2201 # protect os.system from UNC paths on Windows, which it can't handle:
2205 2202 if sys.platform == 'win32':
2206 2203 from IPython.utils._process_win32 import AvoidUNCPath
2207 2204 with AvoidUNCPath() as path:
2208 2205 if path is not None:
2209 2206 cmd = '"pushd %s &&"%s' % (path, cmd)
2210 2207 cmd = py3compat.unicode_to_str(cmd)
2211 2208 try:
2212 2209 ec = os.system(cmd)
2213 2210 except KeyboardInterrupt:
2214 2211 print('\n' + self.get_exception_only(), file=sys.stderr)
2215 2212 ec = -2
2216 2213 else:
2217 2214 cmd = py3compat.unicode_to_str(cmd)
2218 2215 # For posix the result of the subprocess.call() below is an exit
2219 2216 # code, which by convention is zero for success, positive for
2220 2217 # program failure. Exit codes above 128 are reserved for signals,
2221 2218 # and the formula for converting a signal to an exit code is usually
2222 2219 # signal_number+128. To more easily differentiate between exit
2223 2220 # codes and signals, ipython uses negative numbers. For instance
2224 2221 # since control-c is signal 2 but exit code 130, ipython's
2225 2222 # _exit_code variable will read -2. Note that some shells like
2226 2223 # csh and fish don't follow sh/bash conventions for exit codes.
2227 2224 executable = os.environ.get('SHELL', None)
2228 2225 try:
2229 2226 # Use env shell instead of default /bin/sh
2230 2227 ec = subprocess.call(cmd, shell=True, executable=executable)
2231 2228 except KeyboardInterrupt:
2232 2229 # intercept control-C; a long traceback is not useful here
2233 2230 print('\n' + self.get_exception_only(), file=sys.stderr)
2234 2231 ec = 130
2235 2232 if ec > 128:
2236 2233 ec = -(ec - 128)
2237 2234
2238 2235 # We explicitly do NOT return the subprocess status code, because
2239 2236 # a non-None value would trigger :func:`sys.displayhook` calls.
2240 2237 # Instead, we store the exit_code in user_ns. Note the semantics
2241 2238 # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
2242 2239 # but raising SystemExit(_exit_code) will give status 254!
2243 2240 self.user_ns['_exit_code'] = ec
2244 2241
2245 2242 # use piped system by default, because it is better behaved
2246 2243 system = system_piped
2247 2244
2248 2245 def getoutput(self, cmd, split=True, depth=0):
2249 2246 """Get output (possibly including stderr) from a subprocess.
2250 2247
2251 2248 Parameters
2252 2249 ----------
2253 2250 cmd : str
2254 2251 Command to execute (can not end in '&', as background processes are
2255 2252 not supported.
2256 2253 split : bool, optional
2257 2254 If True, split the output into an IPython SList. Otherwise, an
2258 2255 IPython LSString is returned. These are objects similar to normal
2259 2256 lists and strings, with a few convenience attributes for easier
2260 2257 manipulation of line-based output. You can use '?' on them for
2261 2258 details.
2262 2259 depth : int, optional
2263 2260 How many frames above the caller are the local variables which should
2264 2261 be expanded in the command string? The default (0) assumes that the
2265 2262 expansion variables are in the stack frame calling this function.
2266 2263 """
2267 2264 if cmd.rstrip().endswith('&'):
2268 2265 # this is *far* from a rigorous test
2269 2266 raise OSError("Background processes not supported.")
2270 2267 out = getoutput(self.var_expand(cmd, depth=depth+1))
2271 2268 if split:
2272 2269 out = SList(out.splitlines())
2273 2270 else:
2274 2271 out = LSString(out)
2275 2272 return out
2276 2273
2277 2274 #-------------------------------------------------------------------------
2278 2275 # Things related to aliases
2279 2276 #-------------------------------------------------------------------------
2280 2277
2281 2278 def init_alias(self):
2282 2279 self.alias_manager = AliasManager(shell=self, parent=self)
2283 2280 self.configurables.append(self.alias_manager)
2284 2281
2285 2282 #-------------------------------------------------------------------------
2286 2283 # Things related to extensions
2287 2284 #-------------------------------------------------------------------------
2288 2285
2289 2286 def init_extension_manager(self):
2290 2287 self.extension_manager = ExtensionManager(shell=self, parent=self)
2291 2288 self.configurables.append(self.extension_manager)
2292 2289
2293 2290 #-------------------------------------------------------------------------
2294 2291 # Things related to payloads
2295 2292 #-------------------------------------------------------------------------
2296 2293
2297 2294 def init_payload(self):
2298 2295 self.payload_manager = PayloadManager(parent=self)
2299 2296 self.configurables.append(self.payload_manager)
2300 2297
2301 2298 #-------------------------------------------------------------------------
2302 2299 # Things related to the prefilter
2303 2300 #-------------------------------------------------------------------------
2304 2301
2305 2302 def init_prefilter(self):
2306 2303 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2307 2304 self.configurables.append(self.prefilter_manager)
2308 2305 # Ultimately this will be refactored in the new interpreter code, but
2309 2306 # for now, we should expose the main prefilter method (there's legacy
2310 2307 # code out there that may rely on this).
2311 2308 self.prefilter = self.prefilter_manager.prefilter_lines
2312 2309
2313 2310 def auto_rewrite_input(self, cmd):
2314 2311 """Print to the screen the rewritten form of the user's command.
2315 2312
2316 2313 This shows visual feedback by rewriting input lines that cause
2317 2314 automatic calling to kick in, like::
2318 2315
2319 2316 /f x
2320 2317
2321 2318 into::
2322 2319
2323 2320 ------> f(x)
2324 2321
2325 2322 after the user's input prompt. This helps the user understand that the
2326 2323 input line was transformed automatically by IPython.
2327 2324 """
2328 2325 if not self.show_rewritten_input:
2329 2326 return
2330 2327
2331 2328 # This is overridden in TerminalInteractiveShell to use fancy prompts
2332 2329 print("------> " + cmd)
2333 2330
2334 2331 #-------------------------------------------------------------------------
2335 2332 # Things related to extracting values/expressions from kernel and user_ns
2336 2333 #-------------------------------------------------------------------------
2337 2334
2338 2335 def _user_obj_error(self):
2339 2336 """return simple exception dict
2340 2337
2341 2338 for use in user_expressions
2342 2339 """
2343 2340
2344 2341 etype, evalue, tb = self._get_exc_info()
2345 2342 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2346 2343
2347 2344 exc_info = {
2348 2345 u'status' : 'error',
2349 2346 u'traceback' : stb,
2350 2347 u'ename' : unicode_type(etype.__name__),
2351 2348 u'evalue' : py3compat.safe_unicode(evalue),
2352 2349 }
2353 2350
2354 2351 return exc_info
2355 2352
2356 2353 def _format_user_obj(self, obj):
2357 2354 """format a user object to display dict
2358 2355
2359 2356 for use in user_expressions
2360 2357 """
2361 2358
2362 2359 data, md = self.display_formatter.format(obj)
2363 2360 value = {
2364 2361 'status' : 'ok',
2365 2362 'data' : data,
2366 2363 'metadata' : md,
2367 2364 }
2368 2365 return value
2369 2366
2370 2367 def user_expressions(self, expressions):
2371 2368 """Evaluate a dict of expressions in the user's namespace.
2372 2369
2373 2370 Parameters
2374 2371 ----------
2375 2372 expressions : dict
2376 2373 A dict with string keys and string values. The expression values
2377 2374 should be valid Python expressions, each of which will be evaluated
2378 2375 in the user namespace.
2379 2376
2380 2377 Returns
2381 2378 -------
2382 2379 A dict, keyed like the input expressions dict, with the rich mime-typed
2383 2380 display_data of each value.
2384 2381 """
2385 2382 out = {}
2386 2383 user_ns = self.user_ns
2387 2384 global_ns = self.user_global_ns
2388 2385
2389 2386 for key, expr in iteritems(expressions):
2390 2387 try:
2391 2388 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2392 2389 except:
2393 2390 value = self._user_obj_error()
2394 2391 out[key] = value
2395 2392 return out
2396 2393
2397 2394 #-------------------------------------------------------------------------
2398 2395 # Things related to the running of code
2399 2396 #-------------------------------------------------------------------------
2400 2397
2401 2398 def ex(self, cmd):
2402 2399 """Execute a normal python statement in user namespace."""
2403 2400 with self.builtin_trap:
2404 2401 exec(cmd, self.user_global_ns, self.user_ns)
2405 2402
2406 2403 def ev(self, expr):
2407 2404 """Evaluate python expression expr in user namespace.
2408 2405
2409 2406 Returns the result of evaluation
2410 2407 """
2411 2408 with self.builtin_trap:
2412 2409 return eval(expr, self.user_global_ns, self.user_ns)
2413 2410
2414 2411 def safe_execfile(self, fname, *where, **kw):
2415 2412 """A safe version of the builtin execfile().
2416 2413
2417 2414 This version will never throw an exception, but instead print
2418 2415 helpful error messages to the screen. This only works on pure
2419 2416 Python files with the .py extension.
2420 2417
2421 2418 Parameters
2422 2419 ----------
2423 2420 fname : string
2424 2421 The name of the file to be executed.
2425 2422 where : tuple
2426 2423 One or two namespaces, passed to execfile() as (globals,locals).
2427 2424 If only one is given, it is passed as both.
2428 2425 exit_ignore : bool (False)
2429 2426 If True, then silence SystemExit for non-zero status (it is always
2430 2427 silenced for zero status, as it is so common).
2431 2428 raise_exceptions : bool (False)
2432 2429 If True raise exceptions everywhere. Meant for testing.
2433 2430 shell_futures : bool (False)
2434 2431 If True, the code will share future statements with the interactive
2435 2432 shell. It will both be affected by previous __future__ imports, and
2436 2433 any __future__ imports in the code will affect the shell. If False,
2437 2434 __future__ imports are not shared in either direction.
2438 2435
2439 2436 """
2440 2437 kw.setdefault('exit_ignore', False)
2441 2438 kw.setdefault('raise_exceptions', False)
2442 2439 kw.setdefault('shell_futures', False)
2443 2440
2444 2441 fname = os.path.abspath(os.path.expanduser(fname))
2445 2442
2446 2443 # Make sure we can open the file
2447 2444 try:
2448 2445 with open(fname):
2449 2446 pass
2450 2447 except:
2451 2448 warn('Could not open file <%s> for safe execution.' % fname)
2452 2449 return
2453 2450
2454 2451 # Find things also in current directory. This is needed to mimic the
2455 2452 # behavior of running a script from the system command line, where
2456 2453 # Python inserts the script's directory into sys.path
2457 2454 dname = os.path.dirname(fname)
2458 2455
2459 2456 with prepended_to_syspath(dname), self.builtin_trap:
2460 2457 try:
2461 2458 glob, loc = (where + (None, ))[:2]
2462 2459 py3compat.execfile(
2463 2460 fname, glob, loc,
2464 2461 self.compile if kw['shell_futures'] else None)
2465 2462 except SystemExit as status:
2466 2463 # If the call was made with 0 or None exit status (sys.exit(0)
2467 2464 # or sys.exit() ), don't bother showing a traceback, as both of
2468 2465 # these are considered normal by the OS:
2469 2466 # > python -c'import sys;sys.exit(0)'; echo $?
2470 2467 # 0
2471 2468 # > python -c'import sys;sys.exit()'; echo $?
2472 2469 # 0
2473 2470 # For other exit status, we show the exception unless
2474 2471 # explicitly silenced, but only in short form.
2475 2472 if status.code:
2476 2473 if kw['raise_exceptions']:
2477 2474 raise
2478 2475 if not kw['exit_ignore']:
2479 2476 self.showtraceback(exception_only=True)
2480 2477 except:
2481 2478 if kw['raise_exceptions']:
2482 2479 raise
2483 2480 # tb offset is 2 because we wrap execfile
2484 2481 self.showtraceback(tb_offset=2)
2485 2482
2486 2483 def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
2487 2484 """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
2488 2485
2489 2486 Parameters
2490 2487 ----------
2491 2488 fname : str
2492 2489 The name of the file to execute. The filename must have a
2493 2490 .ipy or .ipynb extension.
2494 2491 shell_futures : bool (False)
2495 2492 If True, the code will share future statements with the interactive
2496 2493 shell. It will both be affected by previous __future__ imports, and
2497 2494 any __future__ imports in the code will affect the shell. If False,
2498 2495 __future__ imports are not shared in either direction.
2499 2496 raise_exceptions : bool (False)
2500 2497 If True raise exceptions everywhere. Meant for testing.
2501 2498 """
2502 2499 fname = os.path.abspath(os.path.expanduser(fname))
2503 2500
2504 2501 # Make sure we can open the file
2505 2502 try:
2506 2503 with open(fname):
2507 2504 pass
2508 2505 except:
2509 2506 warn('Could not open file <%s> for safe execution.' % fname)
2510 2507 return
2511 2508
2512 2509 # Find things also in current directory. This is needed to mimic the
2513 2510 # behavior of running a script from the system command line, where
2514 2511 # Python inserts the script's directory into sys.path
2515 2512 dname = os.path.dirname(fname)
2516 2513
2517 2514 def get_cells():
2518 2515 """generator for sequence of code blocks to run"""
2519 2516 if fname.endswith('.ipynb'):
2520 2517 from nbformat import read
2521 2518 with io_open(fname) as f:
2522 2519 nb = read(f, as_version=4)
2523 2520 if not nb.cells:
2524 2521 return
2525 2522 for cell in nb.cells:
2526 2523 if cell.cell_type == 'code':
2527 2524 yield cell.source
2528 2525 else:
2529 2526 with open(fname) as f:
2530 2527 yield f.read()
2531 2528
2532 2529 with prepended_to_syspath(dname):
2533 2530 try:
2534 2531 for cell in get_cells():
2535 2532 result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
2536 2533 if raise_exceptions:
2537 2534 result.raise_error()
2538 2535 elif not result.success:
2539 2536 break
2540 2537 except:
2541 2538 if raise_exceptions:
2542 2539 raise
2543 2540 self.showtraceback()
2544 2541 warn('Unknown failure executing file: <%s>' % fname)
2545 2542
2546 2543 def safe_run_module(self, mod_name, where):
2547 2544 """A safe version of runpy.run_module().
2548 2545
2549 2546 This version will never throw an exception, but instead print
2550 2547 helpful error messages to the screen.
2551 2548
2552 2549 `SystemExit` exceptions with status code 0 or None are ignored.
2553 2550
2554 2551 Parameters
2555 2552 ----------
2556 2553 mod_name : string
2557 2554 The name of the module to be executed.
2558 2555 where : dict
2559 2556 The globals namespace.
2560 2557 """
2561 2558 try:
2562 2559 try:
2563 2560 where.update(
2564 2561 runpy.run_module(str(mod_name), run_name="__main__",
2565 2562 alter_sys=True)
2566 2563 )
2567 2564 except SystemExit as status:
2568 2565 if status.code:
2569 2566 raise
2570 2567 except:
2571 2568 self.showtraceback()
2572 2569 warn('Unknown failure executing module: <%s>' % mod_name)
2573 2570
2574 2571 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2575 2572 """Run a complete IPython cell.
2576 2573
2577 2574 Parameters
2578 2575 ----------
2579 2576 raw_cell : str
2580 2577 The code (including IPython code such as %magic functions) to run.
2581 2578 store_history : bool
2582 2579 If True, the raw and translated cell will be stored in IPython's
2583 2580 history. For user code calling back into IPython's machinery, this
2584 2581 should be set to False.
2585 2582 silent : bool
2586 2583 If True, avoid side-effects, such as implicit displayhooks and
2587 2584 and logging. silent=True forces store_history=False.
2588 2585 shell_futures : bool
2589 2586 If True, the code will share future statements with the interactive
2590 2587 shell. It will both be affected by previous __future__ imports, and
2591 2588 any __future__ imports in the code will affect the shell. If False,
2592 2589 __future__ imports are not shared in either direction.
2593 2590
2594 2591 Returns
2595 2592 -------
2596 2593 result : :class:`ExecutionResult`
2597 2594 """
2598 2595 result = ExecutionResult()
2599 2596
2600 2597 if (not raw_cell) or raw_cell.isspace():
2601 2598 self.last_execution_succeeded = True
2602 2599 return result
2603 2600
2604 2601 if silent:
2605 2602 store_history = False
2606 2603
2607 2604 if store_history:
2608 2605 result.execution_count = self.execution_count
2609 2606
2610 2607 def error_before_exec(value):
2611 2608 result.error_before_exec = value
2612 2609 self.last_execution_succeeded = False
2613 2610 return result
2614 2611
2615 2612 self.events.trigger('pre_execute')
2616 2613 if not silent:
2617 2614 self.events.trigger('pre_run_cell')
2618 2615
2619 2616 # If any of our input transformation (input_transformer_manager or
2620 2617 # prefilter_manager) raises an exception, we store it in this variable
2621 2618 # so that we can display the error after logging the input and storing
2622 2619 # it in the history.
2623 2620 preprocessing_exc_tuple = None
2624 2621 try:
2625 2622 # Static input transformations
2626 2623 cell = self.input_transformer_manager.transform_cell(raw_cell)
2627 2624 except SyntaxError:
2628 2625 preprocessing_exc_tuple = sys.exc_info()
2629 2626 cell = raw_cell # cell has to exist so it can be stored/logged
2630 2627 else:
2631 2628 if len(cell.splitlines()) == 1:
2632 2629 # Dynamic transformations - only applied for single line commands
2633 2630 with self.builtin_trap:
2634 2631 try:
2635 2632 # use prefilter_lines to handle trailing newlines
2636 2633 # restore trailing newline for ast.parse
2637 2634 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
2638 2635 except Exception:
2639 2636 # don't allow prefilter errors to crash IPython
2640 2637 preprocessing_exc_tuple = sys.exc_info()
2641 2638
2642 2639 # Store raw and processed history
2643 2640 if store_history:
2644 2641 self.history_manager.store_inputs(self.execution_count,
2645 2642 cell, raw_cell)
2646 2643 if not silent:
2647 2644 self.logger.log(cell, raw_cell)
2648 2645
2649 2646 # Display the exception if input processing failed.
2650 2647 if preprocessing_exc_tuple is not None:
2651 2648 self.showtraceback(preprocessing_exc_tuple)
2652 2649 if store_history:
2653 2650 self.execution_count += 1
2654 2651 return error_before_exec(preprocessing_exc_tuple[2])
2655 2652
2656 2653 # Our own compiler remembers the __future__ environment. If we want to
2657 2654 # run code with a separate __future__ environment, use the default
2658 2655 # compiler
2659 2656 compiler = self.compile if shell_futures else CachingCompiler()
2660 2657
2661 2658 with self.builtin_trap:
2662 2659 cell_name = self.compile.cache(cell, self.execution_count)
2663 2660
2664 2661 with self.display_trap:
2665 2662 # Compile to bytecode
2666 2663 try:
2667 2664 code_ast = compiler.ast_parse(cell, filename=cell_name)
2668 2665 except self.custom_exceptions as e:
2669 2666 etype, value, tb = sys.exc_info()
2670 2667 self.CustomTB(etype, value, tb)
2671 2668 return error_before_exec(e)
2672 2669 except IndentationError as e:
2673 2670 self.showindentationerror()
2674 2671 if store_history:
2675 2672 self.execution_count += 1
2676 2673 return error_before_exec(e)
2677 2674 except (OverflowError, SyntaxError, ValueError, TypeError,
2678 2675 MemoryError) as e:
2679 2676 self.showsyntaxerror()
2680 2677 if store_history:
2681 2678 self.execution_count += 1
2682 2679 return error_before_exec(e)
2683 2680
2684 2681 # Apply AST transformations
2685 2682 try:
2686 2683 code_ast = self.transform_ast(code_ast)
2687 2684 except InputRejected as e:
2688 2685 self.showtraceback()
2689 2686 if store_history:
2690 2687 self.execution_count += 1
2691 2688 return error_before_exec(e)
2692 2689
2693 2690 # Give the displayhook a reference to our ExecutionResult so it
2694 2691 # can fill in the output value.
2695 2692 self.displayhook.exec_result = result
2696 2693
2697 2694 # Execute the user code
2698 2695 interactivity = "none" if silent else self.ast_node_interactivity
2699 2696 has_raised = self.run_ast_nodes(code_ast.body, cell_name,
2700 2697 interactivity=interactivity, compiler=compiler, result=result)
2701 2698
2702 2699 self.last_execution_succeeded = not has_raised
2703 2700
2704 2701 # Reset this so later displayed values do not modify the
2705 2702 # ExecutionResult
2706 2703 self.displayhook.exec_result = None
2707 2704
2708 2705 self.events.trigger('post_execute')
2709 2706 if not silent:
2710 2707 self.events.trigger('post_run_cell')
2711 2708
2712 2709 if store_history:
2713 2710 # Write output to the database. Does nothing unless
2714 2711 # history output logging is enabled.
2715 2712 self.history_manager.store_output(self.execution_count)
2716 2713 # Each cell is a *single* input, regardless of how many lines it has
2717 2714 self.execution_count += 1
2718 2715
2719 2716 return result
2720 2717
2721 2718 def transform_ast(self, node):
2722 2719 """Apply the AST transformations from self.ast_transformers
2723 2720
2724 2721 Parameters
2725 2722 ----------
2726 2723 node : ast.Node
2727 2724 The root node to be transformed. Typically called with the ast.Module
2728 2725 produced by parsing user input.
2729 2726
2730 2727 Returns
2731 2728 -------
2732 2729 An ast.Node corresponding to the node it was called with. Note that it
2733 2730 may also modify the passed object, so don't rely on references to the
2734 2731 original AST.
2735 2732 """
2736 2733 for transformer in self.ast_transformers:
2737 2734 try:
2738 2735 node = transformer.visit(node)
2739 2736 except InputRejected:
2740 2737 # User-supplied AST transformers can reject an input by raising
2741 2738 # an InputRejected. Short-circuit in this case so that we
2742 2739 # don't unregister the transform.
2743 2740 raise
2744 2741 except Exception:
2745 2742 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
2746 2743 self.ast_transformers.remove(transformer)
2747 2744
2748 2745 if self.ast_transformers:
2749 2746 ast.fix_missing_locations(node)
2750 2747 return node
2751 2748
2752 2749
2753 2750 def run_ast_nodes(self, nodelist, cell_name, interactivity='last_expr',
2754 2751 compiler=compile, result=None):
2755 2752 """Run a sequence of AST nodes. The execution mode depends on the
2756 2753 interactivity parameter.
2757 2754
2758 2755 Parameters
2759 2756 ----------
2760 2757 nodelist : list
2761 2758 A sequence of AST nodes to run.
2762 2759 cell_name : str
2763 2760 Will be passed to the compiler as the filename of the cell. Typically
2764 2761 the value returned by ip.compile.cache(cell).
2765 2762 interactivity : str
2766 2763 'all', 'last', 'last_expr' or 'none', specifying which nodes should be
2767 2764 run interactively (displaying output from expressions). 'last_expr'
2768 2765 will run the last node interactively only if it is an expression (i.e.
2769 2766 expressions in loops or other blocks are not displayed. Other values
2770 2767 for this parameter will raise a ValueError.
2771 2768 compiler : callable
2772 2769 A function with the same interface as the built-in compile(), to turn
2773 2770 the AST nodes into code objects. Default is the built-in compile().
2774 2771 result : ExecutionResult, optional
2775 2772 An object to store exceptions that occur during execution.
2776 2773
2777 2774 Returns
2778 2775 -------
2779 2776 True if an exception occurred while running code, False if it finished
2780 2777 running.
2781 2778 """
2782 2779 if not nodelist:
2783 2780 return
2784 2781
2785 2782 if interactivity == 'last_expr':
2786 2783 if isinstance(nodelist[-1], ast.Expr):
2787 2784 interactivity = "last"
2788 2785 else:
2789 2786 interactivity = "none"
2790 2787
2791 2788 if interactivity == 'none':
2792 2789 to_run_exec, to_run_interactive = nodelist, []
2793 2790 elif interactivity == 'last':
2794 2791 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
2795 2792 elif interactivity == 'all':
2796 2793 to_run_exec, to_run_interactive = [], nodelist
2797 2794 else:
2798 2795 raise ValueError("Interactivity was %r" % interactivity)
2799 2796
2800 2797 try:
2801 2798 for i, node in enumerate(to_run_exec):
2802 2799 mod = ast.Module([node])
2803 2800 code = compiler(mod, cell_name, "exec")
2804 2801 if self.run_code(code, result):
2805 2802 return True
2806 2803
2807 2804 for i, node in enumerate(to_run_interactive):
2808 2805 mod = ast.Interactive([node])
2809 2806 code = compiler(mod, cell_name, "single")
2810 2807 if self.run_code(code, result):
2811 2808 return True
2812 2809
2813 2810 # Flush softspace
2814 2811 if softspace(sys.stdout, 0):
2815 2812 print()
2816 2813
2817 2814 except:
2818 2815 # It's possible to have exceptions raised here, typically by
2819 2816 # compilation of odd code (such as a naked 'return' outside a
2820 2817 # function) that did parse but isn't valid. Typically the exception
2821 2818 # is a SyntaxError, but it's safest just to catch anything and show
2822 2819 # the user a traceback.
2823 2820
2824 2821 # We do only one try/except outside the loop to minimize the impact
2825 2822 # on runtime, and also because if any node in the node list is
2826 2823 # broken, we should stop execution completely.
2827 2824 if result:
2828 2825 result.error_before_exec = sys.exc_info()[1]
2829 2826 self.showtraceback()
2830 2827 return True
2831 2828
2832 2829 return False
2833 2830
2834 2831 def run_code(self, code_obj, result=None):
2835 2832 """Execute a code object.
2836 2833
2837 2834 When an exception occurs, self.showtraceback() is called to display a
2838 2835 traceback.
2839 2836
2840 2837 Parameters
2841 2838 ----------
2842 2839 code_obj : code object
2843 2840 A compiled code object, to be executed
2844 2841 result : ExecutionResult, optional
2845 2842 An object to store exceptions that occur during execution.
2846 2843
2847 2844 Returns
2848 2845 -------
2849 2846 False : successful execution.
2850 2847 True : an error occurred.
2851 2848 """
2852 2849 # Set our own excepthook in case the user code tries to call it
2853 2850 # directly, so that the IPython crash handler doesn't get triggered
2854 2851 old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
2855 2852
2856 2853 # we save the original sys.excepthook in the instance, in case config
2857 2854 # code (such as magics) needs access to it.
2858 2855 self.sys_excepthook = old_excepthook
2859 2856 outflag = 1 # happens in more places, so it's easier as default
2860 2857 try:
2861 2858 try:
2862 2859 self.hooks.pre_run_code_hook()
2863 2860 #rprint('Running code', repr(code_obj)) # dbg
2864 2861 exec(code_obj, self.user_global_ns, self.user_ns)
2865 2862 finally:
2866 2863 # Reset our crash handler in place
2867 2864 sys.excepthook = old_excepthook
2868 2865 except SystemExit as e:
2869 2866 if result is not None:
2870 2867 result.error_in_exec = e
2871 2868 self.showtraceback(exception_only=True)
2872 2869 warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
2873 2870 except self.custom_exceptions:
2874 2871 etype, value, tb = sys.exc_info()
2875 2872 if result is not None:
2876 2873 result.error_in_exec = value
2877 2874 self.CustomTB(etype, value, tb)
2878 2875 except:
2879 2876 if result is not None:
2880 2877 result.error_in_exec = sys.exc_info()[1]
2881 2878 self.showtraceback()
2882 2879 else:
2883 2880 outflag = 0
2884 2881 return outflag
2885 2882
2886 2883 # For backwards compatibility
2887 2884 runcode = run_code
2888 2885
2889 2886 #-------------------------------------------------------------------------
2890 2887 # Things related to GUI support and pylab
2891 2888 #-------------------------------------------------------------------------
2892 2889
2893 2890 active_eventloop = None
2894 2891
2895 2892 def enable_gui(self, gui=None):
2896 2893 raise NotImplementedError('Implement enable_gui in a subclass')
2897 2894
2898 2895 def enable_matplotlib(self, gui=None):
2899 2896 """Enable interactive matplotlib and inline figure support.
2900 2897
2901 2898 This takes the following steps:
2902 2899
2903 2900 1. select the appropriate eventloop and matplotlib backend
2904 2901 2. set up matplotlib for interactive use with that backend
2905 2902 3. configure formatters for inline figure display
2906 2903 4. enable the selected gui eventloop
2907 2904
2908 2905 Parameters
2909 2906 ----------
2910 2907 gui : optional, string
2911 2908 If given, dictates the choice of matplotlib GUI backend to use
2912 2909 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2913 2910 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2914 2911 matplotlib (as dictated by the matplotlib build-time options plus the
2915 2912 user's matplotlibrc configuration file). Note that not all backends
2916 2913 make sense in all contexts, for example a terminal ipython can't
2917 2914 display figures inline.
2918 2915 """
2919 2916 from IPython.core import pylabtools as pt
2920 2917 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
2921 2918
2922 2919 if gui != 'inline':
2923 2920 # If we have our first gui selection, store it
2924 2921 if self.pylab_gui_select is None:
2925 2922 self.pylab_gui_select = gui
2926 2923 # Otherwise if they are different
2927 2924 elif gui != self.pylab_gui_select:
2928 2925 print ('Warning: Cannot change to a different GUI toolkit: %s.'
2929 2926 ' Using %s instead.' % (gui, self.pylab_gui_select))
2930 2927 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
2931 2928
2932 2929 pt.activate_matplotlib(backend)
2933 2930 pt.configure_inline_support(self, backend)
2934 2931
2935 2932 # Now we must activate the gui pylab wants to use, and fix %run to take
2936 2933 # plot updates into account
2937 2934 self.enable_gui(gui)
2938 2935 self.magics_manager.registry['ExecutionMagics'].default_runner = \
2939 2936 pt.mpl_runner(self.safe_execfile)
2940 2937
2941 2938 return gui, backend
2942 2939
2943 2940 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
2944 2941 """Activate pylab support at runtime.
2945 2942
2946 2943 This turns on support for matplotlib, preloads into the interactive
2947 2944 namespace all of numpy and pylab, and configures IPython to correctly
2948 2945 interact with the GUI event loop. The GUI backend to be used can be
2949 2946 optionally selected with the optional ``gui`` argument.
2950 2947
2951 2948 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
2952 2949
2953 2950 Parameters
2954 2951 ----------
2955 2952 gui : optional, string
2956 2953 If given, dictates the choice of matplotlib GUI backend to use
2957 2954 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
2958 2955 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
2959 2956 matplotlib (as dictated by the matplotlib build-time options plus the
2960 2957 user's matplotlibrc configuration file). Note that not all backends
2961 2958 make sense in all contexts, for example a terminal ipython can't
2962 2959 display figures inline.
2963 2960 import_all : optional, bool, default: True
2964 2961 Whether to do `from numpy import *` and `from pylab import *`
2965 2962 in addition to module imports.
2966 2963 welcome_message : deprecated
2967 2964 This argument is ignored, no welcome message will be displayed.
2968 2965 """
2969 2966 from IPython.core.pylabtools import import_pylab
2970 2967
2971 2968 gui, backend = self.enable_matplotlib(gui)
2972 2969
2973 2970 # We want to prevent the loading of pylab to pollute the user's
2974 2971 # namespace as shown by the %who* magics, so we execute the activation
2975 2972 # code in an empty namespace, and we update *both* user_ns and
2976 2973 # user_ns_hidden with this information.
2977 2974 ns = {}
2978 2975 import_pylab(ns, import_all)
2979 2976 # warn about clobbered names
2980 2977 ignored = {"__builtins__"}
2981 2978 both = set(ns).intersection(self.user_ns).difference(ignored)
2982 2979 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
2983 2980 self.user_ns.update(ns)
2984 2981 self.user_ns_hidden.update(ns)
2985 2982 return gui, backend, clobbered
2986 2983
2987 2984 #-------------------------------------------------------------------------
2988 2985 # Utilities
2989 2986 #-------------------------------------------------------------------------
2990 2987
2991 2988 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
2992 2989 """Expand python variables in a string.
2993 2990
2994 2991 The depth argument indicates how many frames above the caller should
2995 2992 be walked to look for the local namespace where to expand variables.
2996 2993
2997 2994 The global namespace for expansion is always the user's interactive
2998 2995 namespace.
2999 2996 """
3000 2997 ns = self.user_ns.copy()
3001 2998 try:
3002 2999 frame = sys._getframe(depth+1)
3003 3000 except ValueError:
3004 3001 # This is thrown if there aren't that many frames on the stack,
3005 3002 # e.g. if a script called run_line_magic() directly.
3006 3003 pass
3007 3004 else:
3008 3005 ns.update(frame.f_locals)
3009 3006
3010 3007 try:
3011 3008 # We have to use .vformat() here, because 'self' is a valid and common
3012 3009 # name, and expanding **ns for .format() would make it collide with
3013 3010 # the 'self' argument of the method.
3014 3011 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
3015 3012 except Exception:
3016 3013 # if formatter couldn't format, just let it go untransformed
3017 3014 pass
3018 3015 return cmd
3019 3016
3020 3017 def mktempfile(self, data=None, prefix='ipython_edit_'):
3021 3018 """Make a new tempfile and return its filename.
3022 3019
3023 3020 This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
3024 3021 but it registers the created filename internally so ipython cleans it up
3025 3022 at exit time.
3026 3023
3027 3024 Optional inputs:
3028 3025
3029 3026 - data(None): if data is given, it gets written out to the temp file
3030 3027 immediately, and the file is closed again."""
3031 3028
3032 3029 dirname = tempfile.mkdtemp(prefix=prefix)
3033 3030 self.tempdirs.append(dirname)
3034 3031
3035 3032 handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
3036 3033 os.close(handle) # On Windows, there can only be one open handle on a file
3037 3034 self.tempfiles.append(filename)
3038 3035
3039 3036 if data:
3040 3037 tmp_file = open(filename,'w')
3041 3038 tmp_file.write(data)
3042 3039 tmp_file.close()
3043 3040 return filename
3044 3041
3045 3042 @undoc
3046 3043 def write(self,data):
3047 3044 """DEPRECATED: Write a string to the default output"""
3048 3045 warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
3049 3046 DeprecationWarning, stacklevel=2)
3050 3047 sys.stdout.write(data)
3051 3048
3052 3049 @undoc
3053 3050 def write_err(self,data):
3054 3051 """DEPRECATED: Write a string to the default error output"""
3055 3052 warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
3056 3053 DeprecationWarning, stacklevel=2)
3057 3054 sys.stderr.write(data)
3058 3055
3059 3056 def ask_yes_no(self, prompt, default=None, interrupt=None):
3060 3057 if self.quiet:
3061 3058 return True
3062 3059 return ask_yes_no(prompt,default,interrupt)
3063 3060
3064 3061 def show_usage(self):
3065 3062 """Show a usage message"""
3066 3063 page.page(IPython.core.usage.interactive_usage)
3067 3064
3068 3065 def extract_input_lines(self, range_str, raw=False):
3069 3066 """Return as a string a set of input history slices.
3070 3067
3071 3068 Parameters
3072 3069 ----------
3073 3070 range_str : string
3074 3071 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3075 3072 since this function is for use by magic functions which get their
3076 3073 arguments as strings. The number before the / is the session
3077 3074 number: ~n goes n back from the current session.
3078 3075
3079 3076 raw : bool, optional
3080 3077 By default, the processed input is used. If this is true, the raw
3081 3078 input history is used instead.
3082 3079
3083 3080 Notes
3084 3081 -----
3085 3082
3086 3083 Slices can be described with two notations:
3087 3084
3088 3085 * ``N:M`` -> standard python form, means including items N...(M-1).
3089 3086 * ``N-M`` -> include items N..M (closed endpoint).
3090 3087 """
3091 3088 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3092 3089 return "\n".join(x for _, _, x in lines)
3093 3090
3094 3091 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
3095 3092 """Get a code string from history, file, url, or a string or macro.
3096 3093
3097 3094 This is mainly used by magic functions.
3098 3095
3099 3096 Parameters
3100 3097 ----------
3101 3098
3102 3099 target : str
3103 3100
3104 3101 A string specifying code to retrieve. This will be tried respectively
3105 3102 as: ranges of input history (see %history for syntax), url,
3106 3103 corresponding .py file, filename, or an expression evaluating to a
3107 3104 string or Macro in the user namespace.
3108 3105
3109 3106 raw : bool
3110 3107 If true (default), retrieve raw history. Has no effect on the other
3111 3108 retrieval mechanisms.
3112 3109
3113 3110 py_only : bool (default False)
3114 3111 Only try to fetch python code, do not try alternative methods to decode file
3115 3112 if unicode fails.
3116 3113
3117 3114 Returns
3118 3115 -------
3119 3116 A string of code.
3120 3117
3121 3118 ValueError is raised if nothing is found, and TypeError if it evaluates
3122 3119 to an object of another type. In each case, .args[0] is a printable
3123 3120 message.
3124 3121 """
3125 3122 code = self.extract_input_lines(target, raw=raw) # Grab history
3126 3123 if code:
3127 3124 return code
3128 3125 try:
3129 3126 if target.startswith(('http://', 'https://')):
3130 3127 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
3131 3128 except UnicodeDecodeError:
3132 3129 if not py_only :
3133 3130 # Deferred import
3134 3131 try:
3135 3132 from urllib.request import urlopen # Py3
3136 3133 except ImportError:
3137 3134 from urllib import urlopen
3138 3135 response = urlopen(target)
3139 3136 return response.read().decode('latin1')
3140 3137 raise ValueError(("'%s' seem to be unreadable.") % target)
3141 3138
3142 3139 potential_target = [target]
3143 3140 try :
3144 3141 potential_target.insert(0,get_py_filename(target))
3145 3142 except IOError:
3146 3143 pass
3147 3144
3148 3145 for tgt in potential_target :
3149 3146 if os.path.isfile(tgt): # Read file
3150 3147 try :
3151 3148 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3152 3149 except UnicodeDecodeError :
3153 3150 if not py_only :
3154 3151 with io_open(tgt,'r', encoding='latin1') as f :
3155 3152 return f.read()
3156 3153 raise ValueError(("'%s' seem to be unreadable.") % target)
3157 3154 elif os.path.isdir(os.path.expanduser(tgt)):
3158 3155 raise ValueError("'%s' is a directory, not a regular file." % target)
3159 3156
3160 3157 if search_ns:
3161 3158 # Inspect namespace to load object source
3162 3159 object_info = self.object_inspect(target, detail_level=1)
3163 3160 if object_info['found'] and object_info['source']:
3164 3161 return object_info['source']
3165 3162
3166 3163 try: # User namespace
3167 3164 codeobj = eval(target, self.user_ns)
3168 3165 except Exception:
3169 3166 raise ValueError(("'%s' was not found in history, as a file, url, "
3170 3167 "nor in the user namespace.") % target)
3171 3168
3172 3169 if isinstance(codeobj, string_types):
3173 3170 return codeobj
3174 3171 elif isinstance(codeobj, Macro):
3175 3172 return codeobj.value
3176 3173
3177 3174 raise TypeError("%s is neither a string nor a macro." % target,
3178 3175 codeobj)
3179 3176
3180 3177 #-------------------------------------------------------------------------
3181 3178 # Things related to IPython exiting
3182 3179 #-------------------------------------------------------------------------
3183 3180 def atexit_operations(self):
3184 3181 """This will be executed at the time of exit.
3185 3182
3186 3183 Cleanup operations and saving of persistent data that is done
3187 3184 unconditionally by IPython should be performed here.
3188 3185
3189 3186 For things that may depend on startup flags or platform specifics (such
3190 3187 as having readline or not), register a separate atexit function in the
3191 3188 code that has the appropriate information, rather than trying to
3192 3189 clutter
3193 3190 """
3194 3191 # Close the history session (this stores the end time and line count)
3195 3192 # this must be *before* the tempfile cleanup, in case of temporary
3196 3193 # history db
3197 3194 self.history_manager.end_session()
3198 3195
3199 3196 # Cleanup all tempfiles and folders left around
3200 3197 for tfile in self.tempfiles:
3201 3198 try:
3202 3199 os.unlink(tfile)
3203 3200 except OSError:
3204 3201 pass
3205 3202
3206 3203 for tdir in self.tempdirs:
3207 3204 try:
3208 3205 os.rmdir(tdir)
3209 3206 except OSError:
3210 3207 pass
3211 3208
3212 3209 # Clear all user namespaces to release all references cleanly.
3213 3210 self.reset(new_session=False)
3214 3211
3215 3212 # Run user hooks
3216 3213 self.hooks.shutdown_hook()
3217 3214
3218 3215 def cleanup(self):
3219 3216 self.restore_sys_module_state()
3220 3217
3221 3218
3222 3219 # Overridden in terminal subclass to change prompts
3223 3220 def switch_doctest_mode(self, mode):
3224 3221 pass
3225 3222
3226 3223
3227 3224 class InteractiveShellABC(with_metaclass(abc.ABCMeta, object)):
3228 3225 """An abstract base class for InteractiveShell."""
3229 3226
3230 3227 InteractiveShellABC.register(InteractiveShell)
@@ -1,806 +1,799 b''
1 1 # encoding: utf-8
2 2 """Tests for the IPython tab-completion machinery."""
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7 import os
8 8 import sys
9 9 import unittest
10 10
11 11 from contextlib import contextmanager
12 12
13 13 import nose.tools as nt
14 14
15 15 from traitlets.config.loader import Config
16 16 from IPython import get_ipython
17 17 from IPython.core import completer
18 18 from IPython.external.decorators import knownfailureif
19 19 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
20 20 from IPython.utils.generics import complete_object
21 21 from IPython.utils.py3compat import string_types, unicode_type
22 22 from IPython.testing import decorators as dec
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Test functions
26 26 #-----------------------------------------------------------------------------
27 27
28 28 @contextmanager
29 29 def greedy_completion():
30 30 ip = get_ipython()
31 31 greedy_original = ip.Completer.greedy
32 32 try:
33 33 ip.Completer.greedy = True
34 34 yield
35 35 finally:
36 36 ip.Completer.greedy = greedy_original
37 37
38 38 def test_protect_filename():
39 39 if sys.platform == 'win32':
40 40 pairs = [('abc','abc'),
41 41 (' abc','" abc"'),
42 42 ('a bc','"a bc"'),
43 43 ('a bc','"a bc"'),
44 44 (' bc','" bc"'),
45 45 ]
46 46 else:
47 47 pairs = [('abc','abc'),
48 48 (' abc',r'\ abc'),
49 49 ('a bc',r'a\ bc'),
50 50 ('a bc',r'a\ \ bc'),
51 51 (' bc',r'\ \ bc'),
52 52 # On posix, we also protect parens and other special characters.
53 53 ('a(bc',r'a\(bc'),
54 54 ('a)bc',r'a\)bc'),
55 55 ('a( )bc',r'a\(\ \)bc'),
56 56 ('a[1]bc', r'a\[1\]bc'),
57 57 ('a{1}bc', r'a\{1\}bc'),
58 58 ('a#bc', r'a\#bc'),
59 59 ('a?bc', r'a\?bc'),
60 60 ('a=bc', r'a\=bc'),
61 61 ('a\\bc', r'a\\bc'),
62 62 ('a|bc', r'a\|bc'),
63 63 ('a;bc', r'a\;bc'),
64 64 ('a:bc', r'a\:bc'),
65 65 ("a'bc", r"a\'bc"),
66 66 ('a*bc', r'a\*bc'),
67 67 ('a"bc', r'a\"bc'),
68 68 ('a^bc', r'a\^bc'),
69 69 ('a&bc', r'a\&bc'),
70 70 ]
71 71 # run the actual tests
72 72 for s1, s2 in pairs:
73 73 s1p = completer.protect_filename(s1)
74 74 nt.assert_equal(s1p, s2)
75 75
76 76
77 77 def check_line_split(splitter, test_specs):
78 78 for part1, part2, split in test_specs:
79 79 cursor_pos = len(part1)
80 80 line = part1+part2
81 81 out = splitter.split_line(line, cursor_pos)
82 82 nt.assert_equal(out, split)
83 83
84 84
85 85 def test_line_split():
86 86 """Basic line splitter test with default specs."""
87 87 sp = completer.CompletionSplitter()
88 88 # The format of the test specs is: part1, part2, expected answer. Parts 1
89 89 # and 2 are joined into the 'line' sent to the splitter, as if the cursor
90 90 # was at the end of part1. So an empty part2 represents someone hitting
91 91 # tab at the end of the line, the most common case.
92 92 t = [('run some/scrip', '', 'some/scrip'),
93 93 ('run scripts/er', 'ror.py foo', 'scripts/er'),
94 94 ('echo $HOM', '', 'HOM'),
95 95 ('print sys.pa', '', 'sys.pa'),
96 96 ('print(sys.pa', '', 'sys.pa'),
97 97 ("execfile('scripts/er", '', 'scripts/er'),
98 98 ('a[x.', '', 'x.'),
99 99 ('a[x.', 'y', 'x.'),
100 100 ('cd "some_file/', '', 'some_file/'),
101 101 ]
102 102 check_line_split(sp, t)
103 103 # Ensure splitting works OK with unicode by re-running the tests with
104 104 # all inputs turned into unicode
105 105 check_line_split(sp, [ map(unicode_type, p) for p in t] )
106 106
107 107
108 108 def test_custom_completion_error():
109 109 """Test that errors from custom attribute completers are silenced."""
110 110 ip = get_ipython()
111 111 class A(object): pass
112 112 ip.user_ns['a'] = A()
113 113
114 114 @complete_object.when_type(A)
115 115 def complete_A(a, existing_completions):
116 116 raise TypeError("this should be silenced")
117 117
118 118 ip.complete("a.")
119 119
120 120
121 121 def test_unicode_completions():
122 122 ip = get_ipython()
123 123 # Some strings that trigger different types of completion. Check them both
124 124 # in str and unicode forms
125 125 s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
126 126 for t in s + list(map(unicode_type, s)):
127 127 # We don't need to check exact completion values (they may change
128 128 # depending on the state of the namespace, but at least no exceptions
129 129 # should be thrown and the return value should be a pair of text, list
130 130 # values.
131 131 text, matches = ip.complete(t)
132 132 nt.assert_true(isinstance(text, string_types))
133 133 nt.assert_true(isinstance(matches, list))
134 134
135 135 def test_latex_completions():
136 136 from IPython.core.latex_symbols import latex_symbols
137 137 import random
138 138 ip = get_ipython()
139 139 # Test some random unicode symbols
140 140 keys = random.sample(latex_symbols.keys(), 10)
141 141 for k in keys:
142 142 text, matches = ip.complete(k)
143 143 nt.assert_equal(len(matches),1)
144 144 nt.assert_equal(text, k)
145 145 nt.assert_equal(matches[0], latex_symbols[k])
146 146 # Test a more complex line
147 147 text, matches = ip.complete(u'print(\\alpha')
148 148 nt.assert_equals(text, u'\\alpha')
149 149 nt.assert_equals(matches[0], latex_symbols['\\alpha'])
150 150 # Test multiple matching latex symbols
151 151 text, matches = ip.complete(u'\\al')
152 152 nt.assert_in('\\alpha', matches)
153 153 nt.assert_in('\\aleph', matches)
154 154
155 155
156 156
157 157
158 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
159 158 def test_back_latex_completion():
160 159 ip = get_ipython()
161 160
162 161 # do not return more than 1 matches fro \beta, only the latex one.
163 162 name, matches = ip.complete('\\Ξ²')
164 163 nt.assert_equal(len(matches), 1)
165 164 nt.assert_equal(matches[0], '\\beta')
166 165
167 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
168 166 def test_back_unicode_completion():
169 167 ip = get_ipython()
170 168
171 169 name, matches = ip.complete('\\β…€')
172 170 nt.assert_equal(len(matches), 1)
173 171 nt.assert_equal(matches[0], '\\ROMAN NUMERAL FIVE')
174 172
175 173
176 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
177 174 def test_forward_unicode_completion():
178 175 ip = get_ipython()
179 176
180 177 name, matches = ip.complete('\\ROMAN NUMERAL FIVE')
181 178 nt.assert_equal(len(matches), 1)
182 179 nt.assert_equal(matches[0], 'β…€')
183 180
184 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
185 181 @dec.knownfailureif(sys.platform == 'win32', 'Fails if there is a C:\\j... path')
186 182 def test_no_ascii_back_completion():
187 183 ip = get_ipython()
188 184 with TemporaryWorkingDirectory(): # Avoid any filename completions
189 185 # single ascii letter that don't have yet completions
190 186 for letter in 'jJ' :
191 187 name, matches = ip.complete('\\'+letter)
192 188 nt.assert_equal(matches, [])
193 189
194 190
195 191
196 192
197 193 class CompletionSplitterTestCase(unittest.TestCase):
198 194 def setUp(self):
199 195 self.sp = completer.CompletionSplitter()
200 196
201 197 def test_delim_setting(self):
202 198 self.sp.delims = ' '
203 199 nt.assert_equal(self.sp.delims, ' ')
204 200 nt.assert_equal(self.sp._delim_expr, '[\ ]')
205 201
206 202 def test_spaces(self):
207 203 """Test with only spaces as split chars."""
208 204 self.sp.delims = ' '
209 205 t = [('foo', '', 'foo'),
210 206 ('run foo', '', 'foo'),
211 207 ('run foo', 'bar', 'foo'),
212 208 ]
213 209 check_line_split(self.sp, t)
214 210
215 211
216 212 def test_has_open_quotes1():
217 213 for s in ["'", "'''", "'hi' '"]:
218 214 nt.assert_equal(completer.has_open_quotes(s), "'")
219 215
220 216
221 217 def test_has_open_quotes2():
222 218 for s in ['"', '"""', '"hi" "']:
223 219 nt.assert_equal(completer.has_open_quotes(s), '"')
224 220
225 221
226 222 def test_has_open_quotes3():
227 223 for s in ["''", "''' '''", "'hi' 'ipython'"]:
228 224 nt.assert_false(completer.has_open_quotes(s))
229 225
230 226
231 227 def test_has_open_quotes4():
232 228 for s in ['""', '""" """', '"hi" "ipython"']:
233 229 nt.assert_false(completer.has_open_quotes(s))
234 230
235 231
236 232 @knownfailureif(sys.platform == 'win32', "abspath completions fail on Windows")
237 233 def test_abspath_file_completions():
238 234 ip = get_ipython()
239 235 with TemporaryDirectory() as tmpdir:
240 236 prefix = os.path.join(tmpdir, 'foo')
241 237 suffixes = ['1', '2']
242 238 names = [prefix+s for s in suffixes]
243 239 for n in names:
244 240 open(n, 'w').close()
245 241
246 242 # Check simple completion
247 243 c = ip.complete(prefix)[1]
248 244 nt.assert_equal(c, names)
249 245
250 246 # Now check with a function call
251 247 cmd = 'a = f("%s' % prefix
252 248 c = ip.complete(prefix, cmd)[1]
253 249 comp = [prefix+s for s in suffixes]
254 250 nt.assert_equal(c, comp)
255 251
256 252
257 253 def test_local_file_completions():
258 254 ip = get_ipython()
259 255 with TemporaryWorkingDirectory():
260 256 prefix = './foo'
261 257 suffixes = ['1', '2']
262 258 names = [prefix+s for s in suffixes]
263 259 for n in names:
264 260 open(n, 'w').close()
265 261
266 262 # Check simple completion
267 263 c = ip.complete(prefix)[1]
268 264 nt.assert_equal(c, names)
269 265
270 266 # Now check with a function call
271 267 cmd = 'a = f("%s' % prefix
272 268 c = ip.complete(prefix, cmd)[1]
273 269 comp = set(prefix+s for s in suffixes)
274 270 nt.assert_true(comp.issubset(set(c)))
275 271
276 272
277 273 def test_greedy_completions():
278 274 ip = get_ipython()
279 275 ip.ex('a=list(range(5))')
280 276 _,c = ip.complete('.',line='a[0].')
281 277 nt.assert_false('.real' in c,
282 278 "Shouldn't have completed on a[0]: %s"%c)
283 279 with greedy_completion():
284 280 def _(line, cursor_pos, expect, message):
285 281 _,c = ip.complete('.', line=line, cursor_pos=cursor_pos)
286 282 nt.assert_in(expect, c, message%c)
287 283
288 284 yield _, 'a[0].', 5, 'a[0].real', "Should have completed on a[0].: %s"
289 285 yield _, 'a[0].r', 6, 'a[0].real', "Should have completed on a[0].r: %s"
290 286
291 287 if sys.version_info > (3,4):
292 288 yield _, 'a[0].from_', 10, 'a[0].from_bytes', "Should have completed on a[0].from_: %s"
293 289
294 290
295 291
296 292 def test_omit__names():
297 293 # also happens to test IPCompleter as a configurable
298 294 ip = get_ipython()
299 295 ip._hidden_attr = 1
300 296 ip._x = {}
301 297 c = ip.Completer
302 298 ip.ex('ip=get_ipython()')
303 299 cfg = Config()
304 300 cfg.IPCompleter.omit__names = 0
305 301 c.update_config(cfg)
306 302 s,matches = c.complete('ip.')
307 303 nt.assert_in('ip.__str__', matches)
308 304 nt.assert_in('ip._hidden_attr', matches)
309 305 cfg = Config()
310 306 cfg.IPCompleter.omit__names = 1
311 307 c.update_config(cfg)
312 308 s,matches = c.complete('ip.')
313 309 nt.assert_not_in('ip.__str__', matches)
314 310 nt.assert_in('ip._hidden_attr', matches)
315 311 cfg = Config()
316 312 cfg.IPCompleter.omit__names = 2
317 313 c.update_config(cfg)
318 314 s,matches = c.complete('ip.')
319 315 nt.assert_not_in('ip.__str__', matches)
320 316 nt.assert_not_in('ip._hidden_attr', matches)
321 317 s,matches = c.complete('ip._x.')
322 318 nt.assert_in('ip._x.keys', matches)
323 319 del ip._hidden_attr
324 320
325 321
326 322 def test_limit_to__all__False_ok():
327 323 ip = get_ipython()
328 324 c = ip.Completer
329 325 ip.ex('class D: x=24')
330 326 ip.ex('d=D()')
331 327 cfg = Config()
332 328 cfg.IPCompleter.limit_to__all__ = False
333 329 c.update_config(cfg)
334 330 s, matches = c.complete('d.')
335 331 nt.assert_in('d.x', matches)
336 332
337 333
338 334 def test_get__all__entries_ok():
339 335 class A(object):
340 336 __all__ = ['x', 1]
341 337 words = completer.get__all__entries(A())
342 338 nt.assert_equal(words, ['x'])
343 339
344 340
345 341 def test_get__all__entries_no__all__ok():
346 342 class A(object):
347 343 pass
348 344 words = completer.get__all__entries(A())
349 345 nt.assert_equal(words, [])
350 346
351 347
352 348 def test_func_kw_completions():
353 349 ip = get_ipython()
354 350 c = ip.Completer
355 351 ip.ex('def myfunc(a=1,b=2): return a+b')
356 352 s, matches = c.complete(None, 'myfunc(1,b')
357 353 nt.assert_in('b=', matches)
358 354 # Simulate completing with cursor right after b (pos==10):
359 355 s, matches = c.complete(None, 'myfunc(1,b)', 10)
360 356 nt.assert_in('b=', matches)
361 357 s, matches = c.complete(None, 'myfunc(a="escaped\\")string",b')
362 358 nt.assert_in('b=', matches)
363 359 #builtin function
364 360 s, matches = c.complete(None, 'min(k, k')
365 361 nt.assert_in('key=', matches)
366 362
367 363
368 364 def test_default_arguments_from_docstring():
369 365 ip = get_ipython()
370 366 c = ip.Completer
371 367 kwd = c._default_arguments_from_docstring(
372 368 'min(iterable[, key=func]) -> value')
373 369 nt.assert_equal(kwd, ['key'])
374 370 #with cython type etc
375 371 kwd = c._default_arguments_from_docstring(
376 372 'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
377 373 nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
378 374 #white spaces
379 375 kwd = c._default_arguments_from_docstring(
380 376 '\n Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
381 377 nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
382 378
383 379 def test_line_magics():
384 380 ip = get_ipython()
385 381 c = ip.Completer
386 382 s, matches = c.complete(None, 'lsmag')
387 383 nt.assert_in('%lsmagic', matches)
388 384 s, matches = c.complete(None, '%lsmag')
389 385 nt.assert_in('%lsmagic', matches)
390 386
391 387
392 388 def test_cell_magics():
393 389 from IPython.core.magic import register_cell_magic
394 390
395 391 @register_cell_magic
396 392 def _foo_cellm(line, cell):
397 393 pass
398 394
399 395 ip = get_ipython()
400 396 c = ip.Completer
401 397
402 398 s, matches = c.complete(None, '_foo_ce')
403 399 nt.assert_in('%%_foo_cellm', matches)
404 400 s, matches = c.complete(None, '%%_foo_ce')
405 401 nt.assert_in('%%_foo_cellm', matches)
406 402
407 403
408 404 def test_line_cell_magics():
409 405 from IPython.core.magic import register_line_cell_magic
410 406
411 407 @register_line_cell_magic
412 408 def _bar_cellm(line, cell):
413 409 pass
414 410
415 411 ip = get_ipython()
416 412 c = ip.Completer
417 413
418 414 # The policy here is trickier, see comments in completion code. The
419 415 # returned values depend on whether the user passes %% or not explicitly,
420 416 # and this will show a difference if the same name is both a line and cell
421 417 # magic.
422 418 s, matches = c.complete(None, '_bar_ce')
423 419 nt.assert_in('%_bar_cellm', matches)
424 420 nt.assert_in('%%_bar_cellm', matches)
425 421 s, matches = c.complete(None, '%_bar_ce')
426 422 nt.assert_in('%_bar_cellm', matches)
427 423 nt.assert_in('%%_bar_cellm', matches)
428 424 s, matches = c.complete(None, '%%_bar_ce')
429 425 nt.assert_not_in('%_bar_cellm', matches)
430 426 nt.assert_in('%%_bar_cellm', matches)
431 427
432 428
433 429 def test_magic_completion_order():
434 430
435 431 ip = get_ipython()
436 432 c = ip.Completer
437 433
438 434 # Test ordering of magics and non-magics with the same name
439 435 # We want the non-magic first
440 436
441 437 # Before importing matplotlib, there should only be one option:
442 438
443 439 text, matches = c.complete('mat')
444 440 nt.assert_equal(matches, ["%matplotlib"])
445 441
446 442
447 443 ip.run_cell("matplotlib = 1") # introduce name into namespace
448 444
449 445 # After the import, there should be two options, ordered like this:
450 446 text, matches = c.complete('mat')
451 447 nt.assert_equal(matches, ["matplotlib", "%matplotlib"])
452 448
453 449
454 450 ip.run_cell("timeit = 1") # define a user variable called 'timeit'
455 451
456 452 # Order of user variable and line and cell magics with same name:
457 453 text, matches = c.complete('timeit')
458 454 nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])
459 455
460 456
461 457 def test_dict_key_completion_string():
462 458 """Test dictionary key completion for string keys"""
463 459 ip = get_ipython()
464 460 complete = ip.Completer.complete
465 461
466 462 ip.user_ns['d'] = {'abc': None}
467 463
468 464 # check completion at different stages
469 465 _, matches = complete(line_buffer="d[")
470 466 nt.assert_in("'abc'", matches)
471 467 nt.assert_not_in("'abc']", matches)
472 468
473 469 _, matches = complete(line_buffer="d['")
474 470 nt.assert_in("abc", matches)
475 471 nt.assert_not_in("abc']", matches)
476 472
477 473 _, matches = complete(line_buffer="d['a")
478 474 nt.assert_in("abc", matches)
479 475 nt.assert_not_in("abc']", matches)
480 476
481 477 # check use of different quoting
482 478 _, matches = complete(line_buffer="d[\"")
483 479 nt.assert_in("abc", matches)
484 480 nt.assert_not_in('abc\"]', matches)
485 481
486 482 _, matches = complete(line_buffer="d[\"a")
487 483 nt.assert_in("abc", matches)
488 484 nt.assert_not_in('abc\"]', matches)
489 485
490 486 # check sensitivity to following context
491 487 _, matches = complete(line_buffer="d[]", cursor_pos=2)
492 488 nt.assert_in("'abc'", matches)
493 489
494 490 _, matches = complete(line_buffer="d['']", cursor_pos=3)
495 491 nt.assert_in("abc", matches)
496 492 nt.assert_not_in("abc'", matches)
497 493 nt.assert_not_in("abc']", matches)
498 494
499 495 # check multiple solutions are correctly returned and that noise is not
500 496 ip.user_ns['d'] = {'abc': None, 'abd': None, 'bad': None, object(): None,
501 497 5: None}
502 498
503 499 _, matches = complete(line_buffer="d['a")
504 500 nt.assert_in("abc", matches)
505 501 nt.assert_in("abd", matches)
506 502 nt.assert_not_in("bad", matches)
507 503 assert not any(m.endswith((']', '"', "'")) for m in matches), matches
508 504
509 505 # check escaping and whitespace
510 506 ip.user_ns['d'] = {'a\nb': None, 'a\'b': None, 'a"b': None, 'a word': None}
511 507 _, matches = complete(line_buffer="d['a")
512 508 nt.assert_in("a\\nb", matches)
513 509 nt.assert_in("a\\'b", matches)
514 510 nt.assert_in("a\"b", matches)
515 511 nt.assert_in("a word", matches)
516 512 assert not any(m.endswith((']', '"', "'")) for m in matches), matches
517 513
518 514 # - can complete on non-initial word of the string
519 515 _, matches = complete(line_buffer="d['a w")
520 516 nt.assert_in("word", matches)
521 517
522 518 # - understands quote escaping
523 519 _, matches = complete(line_buffer="d['a\\'")
524 520 nt.assert_in("b", matches)
525 521
526 522 # - default quoting should work like repr
527 523 _, matches = complete(line_buffer="d[")
528 524 nt.assert_in("\"a'b\"", matches)
529 525
530 526 # - when opening quote with ", possible to match with unescaped apostrophe
531 527 _, matches = complete(line_buffer="d[\"a'")
532 528 nt.assert_in("b", matches)
533 529
534 530 # need to not split at delims that readline won't split at
535 531 if '-' not in ip.Completer.splitter.delims:
536 532 ip.user_ns['d'] = {'before-after': None}
537 533 _, matches = complete(line_buffer="d['before-af")
538 534 nt.assert_in('before-after', matches)
539 535
540 536 def test_dict_key_completion_contexts():
541 537 """Test expression contexts in which dict key completion occurs"""
542 538 ip = get_ipython()
543 539 complete = ip.Completer.complete
544 540 d = {'abc': None}
545 541 ip.user_ns['d'] = d
546 542
547 543 class C:
548 544 data = d
549 545 ip.user_ns['C'] = C
550 546 ip.user_ns['get'] = lambda: d
551 547
552 548 def assert_no_completion(**kwargs):
553 549 _, matches = complete(**kwargs)
554 550 nt.assert_not_in('abc', matches)
555 551 nt.assert_not_in('abc\'', matches)
556 552 nt.assert_not_in('abc\']', matches)
557 553 nt.assert_not_in('\'abc\'', matches)
558 554 nt.assert_not_in('\'abc\']', matches)
559 555
560 556 def assert_completion(**kwargs):
561 557 _, matches = complete(**kwargs)
562 558 nt.assert_in("'abc'", matches)
563 559 nt.assert_not_in("'abc']", matches)
564 560
565 561 # no completion after string closed, even if reopened
566 562 assert_no_completion(line_buffer="d['a'")
567 563 assert_no_completion(line_buffer="d[\"a\"")
568 564 assert_no_completion(line_buffer="d['a' + ")
569 565 assert_no_completion(line_buffer="d['a' + '")
570 566
571 567 # completion in non-trivial expressions
572 568 assert_completion(line_buffer="+ d[")
573 569 assert_completion(line_buffer="(d[")
574 570 assert_completion(line_buffer="C.data[")
575 571
576 572 # greedy flag
577 573 def assert_completion(**kwargs):
578 574 _, matches = complete(**kwargs)
579 575 nt.assert_in("get()['abc']", matches)
580 576
581 577 assert_no_completion(line_buffer="get()[")
582 578 with greedy_completion():
583 579 assert_completion(line_buffer="get()[")
584 580 assert_completion(line_buffer="get()['")
585 581 assert_completion(line_buffer="get()['a")
586 582 assert_completion(line_buffer="get()['ab")
587 583 assert_completion(line_buffer="get()['abc")
588 584
589 585
590 586
591 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
592 587 def test_dict_key_completion_bytes():
593 588 """Test handling of bytes in dict key completion"""
594 589 ip = get_ipython()
595 590 complete = ip.Completer.complete
596 591
597 592 ip.user_ns['d'] = {'abc': None, b'abd': None}
598 593
599 594 _, matches = complete(line_buffer="d[")
600 595 nt.assert_in("'abc'", matches)
601 596 nt.assert_in("b'abd'", matches)
602 597
603 598 if False: # not currently implemented
604 599 _, matches = complete(line_buffer="d[b")
605 600 nt.assert_in("b'abd'", matches)
606 601 nt.assert_not_in("b'abc'", matches)
607 602
608 603 _, matches = complete(line_buffer="d[b'")
609 604 nt.assert_in("abd", matches)
610 605 nt.assert_not_in("abc", matches)
611 606
612 607 _, matches = complete(line_buffer="d[B'")
613 608 nt.assert_in("abd", matches)
614 609 nt.assert_not_in("abc", matches)
615 610
616 611 _, matches = complete(line_buffer="d['")
617 612 nt.assert_in("abc", matches)
618 613 nt.assert_not_in("abd", matches)
619 614
620 615
621 @dec.onlyif(sys.version_info[0] < 3, 'This test only applies in Py<3')
622 616 def test_dict_key_completion_unicode_py2():
623 617 """Test handling of unicode in dict key completion"""
624 618 ip = get_ipython()
625 619 complete = ip.Completer.complete
626 620
627 621 ip.user_ns['d'] = {u'abc': None,
628 622 u'a\u05d0b': None}
629 623
630 624 _, matches = complete(line_buffer="d[")
631 625 nt.assert_in("u'abc'", matches)
632 626 nt.assert_in("u'a\\u05d0b'", matches)
633 627
634 628 _, matches = complete(line_buffer="d['a")
635 629 nt.assert_in("abc", matches)
636 630 nt.assert_not_in("a\\u05d0b", matches)
637 631
638 632 _, matches = complete(line_buffer="d[u'a")
639 633 nt.assert_in("abc", matches)
640 634 nt.assert_in("a\\u05d0b", matches)
641 635
642 636 _, matches = complete(line_buffer="d[U'a")
643 637 nt.assert_in("abc", matches)
644 638 nt.assert_in("a\\u05d0b", matches)
645 639
646 640 # query using escape
647 641 if sys.platform != 'win32':
648 642 # Known failure on Windows
649 643 _, matches = complete(line_buffer=u"d[u'a\\u05d0")
650 644 nt.assert_in("u05d0b", matches) # tokenized after \\
651 645
652 646 # query using character
653 647 _, matches = complete(line_buffer=u"d[u'a\u05d0")
654 648 nt.assert_in(u"a\u05d0b", matches)
655 649
656 650 with greedy_completion():
657 651 _, matches = complete(line_buffer="d[")
658 652 nt.assert_in("d[u'abc']", matches)
659 653 nt.assert_in("d[u'a\\u05d0b']", matches)
660 654
661 655 _, matches = complete(line_buffer="d['a")
662 656 nt.assert_in("d['abc']", matches)
663 657 nt.assert_not_in("d[u'a\\u05d0b']", matches)
664 658
665 659 _, matches = complete(line_buffer="d[u'a")
666 660 nt.assert_in("d[u'abc']", matches)
667 661 nt.assert_in("d[u'a\\u05d0b']", matches)
668 662
669 663 _, matches = complete(line_buffer="d[U'a")
670 664 nt.assert_in("d[U'abc']", matches)
671 665 nt.assert_in("d[U'a\\u05d0b']", matches)
672 666
673 667 # query using escape
674 668 _, matches = complete(line_buffer=u"d[u'a\\u05d0")
675 669 nt.assert_in("d[u'a\\u05d0b']", matches) # tokenized after \\
676 670
677 671 # query using character
678 672 _, matches = complete(line_buffer=u"d[u'a\u05d0")
679 673 nt.assert_in(u"d[u'a\u05d0b']", matches)
680 674
681 675
682 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
683 676 def test_dict_key_completion_unicode_py3():
684 677 """Test handling of unicode in dict key completion"""
685 678 ip = get_ipython()
686 679 complete = ip.Completer.complete
687 680
688 681 ip.user_ns['d'] = {u'a\u05d0': None}
689 682
690 683 # query using escape
691 684 if sys.platform != 'win32':
692 685 # Known failure on Windows
693 686 _, matches = complete(line_buffer="d['a\\u05d0")
694 687 nt.assert_in("u05d0", matches) # tokenized after \\
695 688
696 689 # query using character
697 690 _, matches = complete(line_buffer="d['a\u05d0")
698 691 nt.assert_in(u"a\u05d0", matches)
699 692
700 693 with greedy_completion():
701 694 # query using escape
702 695 _, matches = complete(line_buffer="d['a\\u05d0")
703 696 nt.assert_in("d['a\\u05d0']", matches) # tokenized after \\
704 697
705 698 # query using character
706 699 _, matches = complete(line_buffer="d['a\u05d0")
707 700 nt.assert_in(u"d['a\u05d0']", matches)
708 701
709 702
710 703
711 704 @dec.skip_without('numpy')
712 705 def test_struct_array_key_completion():
713 706 """Test dict key completion applies to numpy struct arrays"""
714 707 import numpy
715 708 ip = get_ipython()
716 709 complete = ip.Completer.complete
717 710 ip.user_ns['d'] = numpy.array([], dtype=[('hello', 'f'), ('world', 'f')])
718 711 _, matches = complete(line_buffer="d['")
719 712 nt.assert_in("hello", matches)
720 713 nt.assert_in("world", matches)
721 714 # complete on the numpy struct itself
722 715 dt = numpy.dtype([('my_head', [('my_dt', '>u4'), ('my_df', '>u4')]),
723 716 ('my_data', '>f4', 5)])
724 717 x = numpy.zeros(2, dtype=dt)
725 718 ip.user_ns['d'] = x[1]
726 719 _, matches = complete(line_buffer="d['")
727 720 nt.assert_in("my_head", matches)
728 721 nt.assert_in("my_data", matches)
729 722 # complete on a nested level
730 723 with greedy_completion():
731 724 ip.user_ns['d'] = numpy.zeros(2, dtype=dt)
732 725 _, matches = complete(line_buffer="d[1]['my_head']['")
733 726 nt.assert_true(any(["my_dt" in m for m in matches]))
734 727 nt.assert_true(any(["my_df" in m for m in matches]))
735 728
736 729
737 730 @dec.skip_without('pandas')
738 731 def test_dataframe_key_completion():
739 732 """Test dict key completion applies to pandas DataFrames"""
740 733 import pandas
741 734 ip = get_ipython()
742 735 complete = ip.Completer.complete
743 736 ip.user_ns['d'] = pandas.DataFrame({'hello': [1], 'world': [2]})
744 737 _, matches = complete(line_buffer="d['")
745 738 nt.assert_in("hello", matches)
746 739 nt.assert_in("world", matches)
747 740
748 741
749 742 def test_dict_key_completion_invalids():
750 743 """Smoke test cases dict key completion can't handle"""
751 744 ip = get_ipython()
752 745 complete = ip.Completer.complete
753 746
754 747 ip.user_ns['no_getitem'] = None
755 748 ip.user_ns['no_keys'] = []
756 749 ip.user_ns['cant_call_keys'] = dict
757 750 ip.user_ns['empty'] = {}
758 751 ip.user_ns['d'] = {'abc': 5}
759 752
760 753 _, matches = complete(line_buffer="no_getitem['")
761 754 _, matches = complete(line_buffer="no_keys['")
762 755 _, matches = complete(line_buffer="cant_call_keys['")
763 756 _, matches = complete(line_buffer="empty['")
764 757 _, matches = complete(line_buffer="name_error['")
765 758 _, matches = complete(line_buffer="d['\\") # incomplete escape
766 759
767 760 class KeyCompletable(object):
768 761 def __init__(self, things=()):
769 762 self.things = things
770 763
771 764 def _ipython_key_completions_(self):
772 765 return list(self.things)
773 766
774 767 def test_object_key_completion():
775 768 ip = get_ipython()
776 769 ip.user_ns['key_completable'] = KeyCompletable(['qwerty', 'qwick'])
777 770
778 771 _, matches = ip.Completer.complete(line_buffer="key_completable['qw")
779 772 nt.assert_in('qwerty', matches)
780 773 nt.assert_in('qwick', matches)
781 774
782 775
783 776 def test_aimport_module_completer():
784 777 ip = get_ipython()
785 778 _, matches = ip.complete('i', '%aimport i')
786 779 nt.assert_in('io', matches)
787 780 nt.assert_not_in('int', matches)
788 781
789 782 def test_nested_import_module_completer():
790 783 ip = get_ipython()
791 784 _, matches = ip.complete(None, 'import IPython.co', 17)
792 785 nt.assert_in('IPython.core', matches)
793 786 nt.assert_not_in('import IPython.core', matches)
794 787 nt.assert_not_in('IPython.display', matches)
795 788
796 789 def test_import_module_completer():
797 790 ip = get_ipython()
798 791 _, matches = ip.complete('i', 'import i')
799 792 nt.assert_in('io', matches)
800 793 nt.assert_not_in('int', matches)
801 794
802 795 def test_from_module_completer():
803 796 ip = get_ipython()
804 797 _, matches = ip.complete('B', 'from io import B', 16)
805 798 nt.assert_in('BytesIO', matches)
806 799 nt.assert_not_in('BaseException', matches)
@@ -1,950 +1,918 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the key interactiveshell module.
3 3
4 4 Historically the main classes in interactiveshell have been under-tested. This
5 5 module should grow as many single-method tests as possible to trap many of the
6 6 recurring bugs we seem to encounter with high-level interaction.
7 7 """
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 import ast
13 13 import os
14 14 import signal
15 15 import shutil
16 16 import sys
17 17 import tempfile
18 18 import unittest
19 19 try:
20 20 from unittest import mock
21 21 except ImportError:
22 22 import mock
23 23 from os.path import join
24 24
25 25 import nose.tools as nt
26 26
27 27 from IPython.core.error import InputRejected
28 28 from IPython.core.inputtransformer import InputTransformer
29 29 from IPython.testing.decorators import (
30 30 skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
31 31 )
32 32 from IPython.testing import tools as tt
33 33 from IPython.utils.process import find_cmd
34 34 from IPython.utils import py3compat
35 35 from IPython.utils.py3compat import unicode_type, PY3
36 36
37 37 if PY3:
38 38 from io import StringIO
39 39 else:
40 40 from StringIO import StringIO
41 41
42 42 #-----------------------------------------------------------------------------
43 43 # Globals
44 44 #-----------------------------------------------------------------------------
45 45 # This is used by every single test, no point repeating it ad nauseam
46 46 ip = get_ipython()
47 47
48 48 #-----------------------------------------------------------------------------
49 49 # Tests
50 50 #-----------------------------------------------------------------------------
51 51
52 52 class DerivedInterrupt(KeyboardInterrupt):
53 53 pass
54 54
55 55 class InteractiveShellTestCase(unittest.TestCase):
56 56 def test_naked_string_cells(self):
57 57 """Test that cells with only naked strings are fully executed"""
58 58 # First, single-line inputs
59 59 ip.run_cell('"a"\n')
60 60 self.assertEqual(ip.user_ns['_'], 'a')
61 61 # And also multi-line cells
62 62 ip.run_cell('"""a\nb"""\n')
63 63 self.assertEqual(ip.user_ns['_'], 'a\nb')
64 64
65 65 def test_run_empty_cell(self):
66 66 """Just make sure we don't get a horrible error with a blank
67 67 cell of input. Yes, I did overlook that."""
68 68 old_xc = ip.execution_count
69 69 res = ip.run_cell('')
70 70 self.assertEqual(ip.execution_count, old_xc)
71 71 self.assertEqual(res.execution_count, None)
72 72
73 73 def test_run_cell_multiline(self):
74 74 """Multi-block, multi-line cells must execute correctly.
75 75 """
76 76 src = '\n'.join(["x=1",
77 77 "y=2",
78 78 "if 1:",
79 79 " x += 1",
80 80 " y += 1",])
81 81 res = ip.run_cell(src)
82 82 self.assertEqual(ip.user_ns['x'], 2)
83 83 self.assertEqual(ip.user_ns['y'], 3)
84 84 self.assertEqual(res.success, True)
85 85 self.assertEqual(res.result, None)
86 86
87 87 def test_multiline_string_cells(self):
88 88 "Code sprinkled with multiline strings should execute (GH-306)"
89 89 ip.run_cell('tmp=0')
90 90 self.assertEqual(ip.user_ns['tmp'], 0)
91 91 res = ip.run_cell('tmp=1;"""a\nb"""\n')
92 92 self.assertEqual(ip.user_ns['tmp'], 1)
93 93 self.assertEqual(res.success, True)
94 94 self.assertEqual(res.result, "a\nb")
95 95
96 96 def test_dont_cache_with_semicolon(self):
97 97 "Ending a line with semicolon should not cache the returned object (GH-307)"
98 98 oldlen = len(ip.user_ns['Out'])
99 99 for cell in ['1;', '1;1;']:
100 100 res = ip.run_cell(cell, store_history=True)
101 101 newlen = len(ip.user_ns['Out'])
102 102 self.assertEqual(oldlen, newlen)
103 103 self.assertIsNone(res.result)
104 104 i = 0
105 105 #also test the default caching behavior
106 106 for cell in ['1', '1;1']:
107 107 ip.run_cell(cell, store_history=True)
108 108 newlen = len(ip.user_ns['Out'])
109 109 i += 1
110 110 self.assertEqual(oldlen+i, newlen)
111 111
112 112 def test_syntax_error(self):
113 113 res = ip.run_cell("raise = 3")
114 114 self.assertIsInstance(res.error_before_exec, SyntaxError)
115 115
116 116 def test_In_variable(self):
117 117 "Verify that In variable grows with user input (GH-284)"
118 118 oldlen = len(ip.user_ns['In'])
119 119 ip.run_cell('1;', store_history=True)
120 120 newlen = len(ip.user_ns['In'])
121 121 self.assertEqual(oldlen+1, newlen)
122 122 self.assertEqual(ip.user_ns['In'][-1],'1;')
123 123
124 124 def test_magic_names_in_string(self):
125 125 ip.run_cell('a = """\n%exit\n"""')
126 126 self.assertEqual(ip.user_ns['a'], '\n%exit\n')
127 127
128 128 def test_trailing_newline(self):
129 129 """test that running !(command) does not raise a SyntaxError"""
130 130 ip.run_cell('!(true)\n', False)
131 131 ip.run_cell('!(true)\n\n\n', False)
132 132
133 133 def test_gh_597(self):
134 134 """Pretty-printing lists of objects with non-ascii reprs may cause
135 135 problems."""
136 136 class Spam(object):
137 137 def __repr__(self):
138 138 return "\xe9"*50
139 139 import IPython.core.formatters
140 140 f = IPython.core.formatters.PlainTextFormatter()
141 141 f([Spam(),Spam()])
142 142
143 143
144 144 def test_future_flags(self):
145 145 """Check that future flags are used for parsing code (gh-777)"""
146 146 ip.run_cell('from __future__ import print_function')
147 147 try:
148 148 ip.run_cell('prfunc_return_val = print(1,2, sep=" ")')
149 149 assert 'prfunc_return_val' in ip.user_ns
150 150 finally:
151 151 # Reset compiler flags so we don't mess up other tests.
152 152 ip.compile.reset_compiler_flags()
153 153
154 154 def test_future_unicode(self):
155 155 """Check that unicode_literals is imported from __future__ (gh #786)"""
156 156 try:
157 157 ip.run_cell(u'byte_str = "a"')
158 158 assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default
159 159 ip.run_cell('from __future__ import unicode_literals')
160 160 ip.run_cell(u'unicode_str = "a"')
161 161 assert isinstance(ip.user_ns['unicode_str'], unicode_type) # strings literals are now unicode
162 162 finally:
163 163 # Reset compiler flags so we don't mess up other tests.
164 164 ip.compile.reset_compiler_flags()
165 165
166 166 def test_can_pickle(self):
167 167 "Can we pickle objects defined interactively (GH-29)"
168 168 ip = get_ipython()
169 169 ip.reset()
170 170 ip.run_cell(("class Mylist(list):\n"
171 171 " def __init__(self,x=[]):\n"
172 172 " list.__init__(self,x)"))
173 173 ip.run_cell("w=Mylist([1,2,3])")
174 174
175 175 from pickle import dumps
176 176
177 177 # We need to swap in our main module - this is only necessary
178 178 # inside the test framework, because IPython puts the interactive module
179 179 # in place (but the test framework undoes this).
180 180 _main = sys.modules['__main__']
181 181 sys.modules['__main__'] = ip.user_module
182 182 try:
183 183 res = dumps(ip.user_ns["w"])
184 184 finally:
185 185 sys.modules['__main__'] = _main
186 186 self.assertTrue(isinstance(res, bytes))
187 187
188 188 def test_global_ns(self):
189 189 "Code in functions must be able to access variables outside them."
190 190 ip = get_ipython()
191 191 ip.run_cell("a = 10")
192 192 ip.run_cell(("def f(x):\n"
193 193 " return x + a"))
194 194 ip.run_cell("b = f(12)")
195 195 self.assertEqual(ip.user_ns["b"], 22)
196 196
197 197 def test_bad_custom_tb(self):
198 198 """Check that InteractiveShell is protected from bad custom exception handlers"""
199 199 ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
200 200 self.assertEqual(ip.custom_exceptions, (IOError,))
201 201 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
202 202 ip.run_cell(u'raise IOError("foo")')
203 203 self.assertEqual(ip.custom_exceptions, ())
204 204
205 205 def test_bad_custom_tb_return(self):
206 206 """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
207 207 ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
208 208 self.assertEqual(ip.custom_exceptions, (NameError,))
209 209 with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
210 210 ip.run_cell(u'a=abracadabra')
211 211 self.assertEqual(ip.custom_exceptions, ())
212 212
213 213 def test_drop_by_id(self):
214 214 myvars = {"a":object(), "b":object(), "c": object()}
215 215 ip.push(myvars, interactive=False)
216 216 for name in myvars:
217 217 assert name in ip.user_ns, name
218 218 assert name in ip.user_ns_hidden, name
219 219 ip.user_ns['b'] = 12
220 220 ip.drop_by_id(myvars)
221 221 for name in ["a", "c"]:
222 222 assert name not in ip.user_ns, name
223 223 assert name not in ip.user_ns_hidden, name
224 224 assert ip.user_ns['b'] == 12
225 225 ip.reset()
226 226
227 227 def test_var_expand(self):
228 228 ip.user_ns['f'] = u'Ca\xf1o'
229 229 self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
230 230 self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
231 231 self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
232 232 self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
233 233
234 234 ip.user_ns['f'] = b'Ca\xc3\xb1o'
235 235 # This should not raise any exception:
236 236 ip.var_expand(u'echo $f')
237 237
238 238 def test_var_expand_local(self):
239 239 """Test local variable expansion in !system and %magic calls"""
240 240 # !system
241 241 ip.run_cell('def test():\n'
242 242 ' lvar = "ttt"\n'
243 243 ' ret = !echo {lvar}\n'
244 244 ' return ret[0]\n')
245 245 res = ip.user_ns['test']()
246 246 nt.assert_in('ttt', res)
247 247
248 248 # %magic
249 249 ip.run_cell('def makemacro():\n'
250 250 ' macroname = "macro_var_expand_locals"\n'
251 251 ' %macro {macroname} codestr\n')
252 252 ip.user_ns['codestr'] = "str(12)"
253 253 ip.run_cell('makemacro()')
254 254 nt.assert_in('macro_var_expand_locals', ip.user_ns)
255 255
256 256 def test_var_expand_self(self):
257 257 """Test variable expansion with the name 'self', which was failing.
258 258
259 259 See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
260 260 """
261 261 ip.run_cell('class cTest:\n'
262 262 ' classvar="see me"\n'
263 263 ' def test(self):\n'
264 264 ' res = !echo Variable: {self.classvar}\n'
265 265 ' return res[0]\n')
266 266 nt.assert_in('see me', ip.user_ns['cTest']().test())
267 267
268 268 def test_bad_var_expand(self):
269 269 """var_expand on invalid formats shouldn't raise"""
270 270 # SyntaxError
271 271 self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
272 272 # NameError
273 273 self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
274 274 # ZeroDivisionError
275 275 self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
276 276
277 277 def test_silent_postexec(self):
278 278 """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
279 279 pre_explicit = mock.Mock()
280 280 pre_always = mock.Mock()
281 281 post_explicit = mock.Mock()
282 282 post_always = mock.Mock()
283 283
284 284 ip.events.register('pre_run_cell', pre_explicit)
285 285 ip.events.register('pre_execute', pre_always)
286 286 ip.events.register('post_run_cell', post_explicit)
287 287 ip.events.register('post_execute', post_always)
288 288
289 289 try:
290 290 ip.run_cell("1", silent=True)
291 291 assert pre_always.called
292 292 assert not pre_explicit.called
293 293 assert post_always.called
294 294 assert not post_explicit.called
295 295 # double-check that non-silent exec did what we expected
296 296 # silent to avoid
297 297 ip.run_cell("1")
298 298 assert pre_explicit.called
299 299 assert post_explicit.called
300 300 finally:
301 301 # remove post-exec
302 302 ip.events.unregister('pre_run_cell', pre_explicit)
303 303 ip.events.unregister('pre_execute', pre_always)
304 304 ip.events.unregister('post_run_cell', post_explicit)
305 305 ip.events.unregister('post_execute', post_always)
306 306
307 307 def test_silent_noadvance(self):
308 308 """run_cell(silent=True) doesn't advance execution_count"""
309 309 ec = ip.execution_count
310 310 # silent should force store_history=False
311 311 ip.run_cell("1", store_history=True, silent=True)
312 312
313 313 self.assertEqual(ec, ip.execution_count)
314 314 # double-check that non-silent exec did what we expected
315 315 # silent to avoid
316 316 ip.run_cell("1", store_history=True)
317 317 self.assertEqual(ec+1, ip.execution_count)
318 318
319 319 def test_silent_nodisplayhook(self):
320 320 """run_cell(silent=True) doesn't trigger displayhook"""
321 321 d = dict(called=False)
322 322
323 323 trap = ip.display_trap
324 324 save_hook = trap.hook
325 325
326 326 def failing_hook(*args, **kwargs):
327 327 d['called'] = True
328 328
329 329 try:
330 330 trap.hook = failing_hook
331 331 res = ip.run_cell("1", silent=True)
332 332 self.assertFalse(d['called'])
333 333 self.assertIsNone(res.result)
334 334 # double-check that non-silent exec did what we expected
335 335 # silent to avoid
336 336 ip.run_cell("1")
337 337 self.assertTrue(d['called'])
338 338 finally:
339 339 trap.hook = save_hook
340 340
341 @skipif(sys.version_info[0] >= 3, "softspace removed in py3")
342 def test_print_softspace(self):
343 """Verify that softspace is handled correctly when executing multiple
344 statements.
345
346 In [1]: print 1; print 2
347 1
348 2
349
350 In [2]: print 1,; print 2
351 1 2
352 """
353
354 341 def test_ofind_line_magic(self):
355 342 from IPython.core.magic import register_line_magic
356 343
357 344 @register_line_magic
358 345 def lmagic(line):
359 346 "A line magic"
360 347
361 348 # Get info on line magic
362 349 lfind = ip._ofind('lmagic')
363 350 info = dict(found=True, isalias=False, ismagic=True,
364 351 namespace = 'IPython internal', obj= lmagic.__wrapped__,
365 352 parent = None)
366 353 nt.assert_equal(lfind, info)
367 354
368 355 def test_ofind_cell_magic(self):
369 356 from IPython.core.magic import register_cell_magic
370 357
371 358 @register_cell_magic
372 359 def cmagic(line, cell):
373 360 "A cell magic"
374 361
375 362 # Get info on cell magic
376 363 find = ip._ofind('cmagic')
377 364 info = dict(found=True, isalias=False, ismagic=True,
378 365 namespace = 'IPython internal', obj= cmagic.__wrapped__,
379 366 parent = None)
380 367 nt.assert_equal(find, info)
381 368
382 369 def test_ofind_property_with_error(self):
383 370 class A(object):
384 371 @property
385 372 def foo(self):
386 373 raise NotImplementedError()
387 374 a = A()
388 375
389 376 found = ip._ofind('a.foo', [('locals', locals())])
390 377 info = dict(found=True, isalias=False, ismagic=False,
391 378 namespace='locals', obj=A.foo, parent=a)
392 379 nt.assert_equal(found, info)
393 380
394 381 def test_ofind_multiple_attribute_lookups(self):
395 382 class A(object):
396 383 @property
397 384 def foo(self):
398 385 raise NotImplementedError()
399 386
400 387 a = A()
401 388 a.a = A()
402 389 a.a.a = A()
403 390
404 391 found = ip._ofind('a.a.a.foo', [('locals', locals())])
405 392 info = dict(found=True, isalias=False, ismagic=False,
406 393 namespace='locals', obj=A.foo, parent=a.a.a)
407 394 nt.assert_equal(found, info)
408 395
409 396 def test_ofind_slotted_attributes(self):
410 397 class A(object):
411 398 __slots__ = ['foo']
412 399 def __init__(self):
413 400 self.foo = 'bar'
414 401
415 402 a = A()
416 403 found = ip._ofind('a.foo', [('locals', locals())])
417 404 info = dict(found=True, isalias=False, ismagic=False,
418 405 namespace='locals', obj=a.foo, parent=a)
419 406 nt.assert_equal(found, info)
420 407
421 408 found = ip._ofind('a.bar', [('locals', locals())])
422 409 info = dict(found=False, isalias=False, ismagic=False,
423 410 namespace=None, obj=None, parent=a)
424 411 nt.assert_equal(found, info)
425 412
426 413 def test_ofind_prefers_property_to_instance_level_attribute(self):
427 414 class A(object):
428 415 @property
429 416 def foo(self):
430 417 return 'bar'
431 418 a = A()
432 419 a.__dict__['foo'] = 'baz'
433 420 nt.assert_equal(a.foo, 'bar')
434 421 found = ip._ofind('a.foo', [('locals', locals())])
435 422 nt.assert_is(found['obj'], A.foo)
436 423
437 424 def test_custom_syntaxerror_exception(self):
438 425 called = []
439 426 def my_handler(shell, etype, value, tb, tb_offset=None):
440 427 called.append(etype)
441 428 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
442 429
443 430 ip.set_custom_exc((SyntaxError,), my_handler)
444 431 try:
445 432 ip.run_cell("1f")
446 433 # Check that this was called, and only once.
447 434 self.assertEqual(called, [SyntaxError])
448 435 finally:
449 436 # Reset the custom exception hook
450 437 ip.set_custom_exc((), None)
451 438
452 439 def test_custom_exception(self):
453 440 called = []
454 441 def my_handler(shell, etype, value, tb, tb_offset=None):
455 442 called.append(etype)
456 443 shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
457 444
458 445 ip.set_custom_exc((ValueError,), my_handler)
459 446 try:
460 447 res = ip.run_cell("raise ValueError('test')")
461 448 # Check that this was called, and only once.
462 449 self.assertEqual(called, [ValueError])
463 450 # Check that the error is on the result object
464 451 self.assertIsInstance(res.error_in_exec, ValueError)
465 452 finally:
466 453 # Reset the custom exception hook
467 454 ip.set_custom_exc((), None)
468 455
469 @skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
470 def test_future_environment(self):
471 "Can we run code with & without the shell's __future__ imports?"
472 ip.run_cell("from __future__ import division")
473 ip.run_cell("a = 1/2", shell_futures=True)
474 self.assertEqual(ip.user_ns['a'], 0.5)
475 ip.run_cell("b = 1/2", shell_futures=False)
476 self.assertEqual(ip.user_ns['b'], 0)
477
478 ip.compile.reset_compiler_flags()
479 # This shouldn't leak to the shell's compiler
480 ip.run_cell("from __future__ import division \nc=1/2", shell_futures=False)
481 self.assertEqual(ip.user_ns['c'], 0.5)
482 ip.run_cell("d = 1/2", shell_futures=True)
483 self.assertEqual(ip.user_ns['d'], 0)
484
485 456 def test_mktempfile(self):
486 457 filename = ip.mktempfile()
487 458 # Check that we can open the file again on Windows
488 459 with open(filename, 'w') as f:
489 460 f.write('abc')
490 461
491 462 filename = ip.mktempfile(data='blah')
492 463 with open(filename, 'r') as f:
493 464 self.assertEqual(f.read(), 'blah')
494 465
495 466 def test_new_main_mod(self):
496 467 # Smoketest to check that this accepts a unicode module name
497 468 name = u'jiefmw'
498 469 mod = ip.new_main_mod(u'%s.py' % name, name)
499 470 self.assertEqual(mod.__name__, name)
500 471
501 472 def test_get_exception_only(self):
502 473 try:
503 474 raise KeyboardInterrupt
504 475 except KeyboardInterrupt:
505 476 msg = ip.get_exception_only()
506 477 self.assertEqual(msg, 'KeyboardInterrupt\n')
507 478
508 479 try:
509 480 raise DerivedInterrupt("foo")
510 481 except KeyboardInterrupt:
511 482 msg = ip.get_exception_only()
512 if sys.version_info[0] <= 2:
513 self.assertEqual(msg, 'DerivedInterrupt: foo\n')
514 else:
515 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
483 self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
516 484
517 485 def test_inspect_text(self):
518 486 ip.run_cell('a = 5')
519 487 text = ip.object_inspect_text('a')
520 488 self.assertIsInstance(text, unicode_type)
521 489
522 490
523 491 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
524 492
525 493 @onlyif_unicode_paths
526 494 def setUp(self):
527 495 self.BASETESTDIR = tempfile.mkdtemp()
528 496 self.TESTDIR = join(self.BASETESTDIR, u"Γ₯Àâ")
529 497 os.mkdir(self.TESTDIR)
530 498 with open(join(self.TESTDIR, u"Γ₯Àâtestscript.py"), "w") as sfile:
531 499 sfile.write("pass\n")
532 500 self.oldpath = py3compat.getcwd()
533 501 os.chdir(self.TESTDIR)
534 502 self.fname = u"Γ₯Àâtestscript.py"
535 503
536 504 def tearDown(self):
537 505 os.chdir(self.oldpath)
538 506 shutil.rmtree(self.BASETESTDIR)
539 507
540 508 @onlyif_unicode_paths
541 509 def test_1(self):
542 510 """Test safe_execfile with non-ascii path
543 511 """
544 512 ip.safe_execfile(self.fname, {}, raise_exceptions=True)
545 513
546 514 class ExitCodeChecks(tt.TempFileMixin):
547 515 def test_exit_code_ok(self):
548 516 self.system('exit 0')
549 517 self.assertEqual(ip.user_ns['_exit_code'], 0)
550 518
551 519 def test_exit_code_error(self):
552 520 self.system('exit 1')
553 521 self.assertEqual(ip.user_ns['_exit_code'], 1)
554 522
555 523 @skipif(not hasattr(signal, 'SIGALRM'))
556 524 def test_exit_code_signal(self):
557 525 self.mktmp("import signal, time\n"
558 526 "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
559 527 "time.sleep(1)\n")
560 528 self.system("%s %s" % (sys.executable, self.fname))
561 529 self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
562 530
563 531 @onlyif_cmds_exist("csh")
564 532 def test_exit_code_signal_csh(self):
565 533 SHELL = os.environ.get('SHELL', None)
566 534 os.environ['SHELL'] = find_cmd("csh")
567 535 try:
568 536 self.test_exit_code_signal()
569 537 finally:
570 538 if SHELL is not None:
571 539 os.environ['SHELL'] = SHELL
572 540 else:
573 541 del os.environ['SHELL']
574 542
575 543 class TestSystemRaw(unittest.TestCase, ExitCodeChecks):
576 544 system = ip.system_raw
577 545
578 546 @onlyif_unicode_paths
579 547 def test_1(self):
580 548 """Test system_raw with non-ascii cmd
581 549 """
582 550 cmd = u'''python -c "'Γ₯Àâ'" '''
583 551 ip.system_raw(cmd)
584 552
585 553 @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
586 554 @mock.patch('os.system', side_effect=KeyboardInterrupt)
587 555 def test_control_c(self, *mocks):
588 556 try:
589 557 self.system("sleep 1 # wont happen")
590 558 except KeyboardInterrupt:
591 559 self.fail("system call should intercept "
592 560 "keyboard interrupt from subprocess.call")
593 561 self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT)
594 562
595 563 # TODO: Exit codes are currently ignored on Windows.
596 564 class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks):
597 565 system = ip.system_piped
598 566
599 567 @skip_win32
600 568 def test_exit_code_ok(self):
601 569 ExitCodeChecks.test_exit_code_ok(self)
602 570
603 571 @skip_win32
604 572 def test_exit_code_error(self):
605 573 ExitCodeChecks.test_exit_code_error(self)
606 574
607 575 @skip_win32
608 576 def test_exit_code_signal(self):
609 577 ExitCodeChecks.test_exit_code_signal(self)
610 578
611 579 class TestModules(unittest.TestCase, tt.TempFileMixin):
612 580 def test_extraneous_loads(self):
613 581 """Test we're not loading modules on startup that we shouldn't.
614 582 """
615 583 self.mktmp("import sys\n"
616 584 "print('numpy' in sys.modules)\n"
617 585 "print('ipyparallel' in sys.modules)\n"
618 586 "print('ipykernel' in sys.modules)\n"
619 587 )
620 588 out = "False\nFalse\nFalse\n"
621 589 tt.ipexec_validate(self.fname, out)
622 590
623 591 class Negator(ast.NodeTransformer):
624 592 """Negates all number literals in an AST."""
625 593 def visit_Num(self, node):
626 594 node.n = -node.n
627 595 return node
628 596
629 597 class TestAstTransform(unittest.TestCase):
630 598 def setUp(self):
631 599 self.negator = Negator()
632 600 ip.ast_transformers.append(self.negator)
633 601
634 602 def tearDown(self):
635 603 ip.ast_transformers.remove(self.negator)
636 604
637 605 def test_run_cell(self):
638 606 with tt.AssertPrints('-34'):
639 607 ip.run_cell('print (12 + 22)')
640 608
641 609 # A named reference to a number shouldn't be transformed.
642 610 ip.user_ns['n'] = 55
643 611 with tt.AssertNotPrints('-55'):
644 612 ip.run_cell('print (n)')
645 613
646 614 def test_timeit(self):
647 615 called = set()
648 616 def f(x):
649 617 called.add(x)
650 618 ip.push({'f':f})
651 619
652 620 with tt.AssertPrints("average of "):
653 621 ip.run_line_magic("timeit", "-n1 f(1)")
654 622 self.assertEqual(called, {-1})
655 623 called.clear()
656 624
657 625 with tt.AssertPrints("average of "):
658 626 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
659 627 self.assertEqual(called, {-2, -3})
660 628
661 629 def test_time(self):
662 630 called = []
663 631 def f(x):
664 632 called.append(x)
665 633 ip.push({'f':f})
666 634
667 635 # Test with an expression
668 636 with tt.AssertPrints("Wall time: "):
669 637 ip.run_line_magic("time", "f(5+9)")
670 638 self.assertEqual(called, [-14])
671 639 called[:] = []
672 640
673 641 # Test with a statement (different code path)
674 642 with tt.AssertPrints("Wall time: "):
675 643 ip.run_line_magic("time", "a = f(-3 + -2)")
676 644 self.assertEqual(called, [5])
677 645
678 646 def test_macro(self):
679 647 ip.push({'a':10})
680 648 # The AST transformation makes this do a+=-1
681 649 ip.define_macro("amacro", "a+=1\nprint(a)")
682 650
683 651 with tt.AssertPrints("9"):
684 652 ip.run_cell("amacro")
685 653 with tt.AssertPrints("8"):
686 654 ip.run_cell("amacro")
687 655
688 656 class IntegerWrapper(ast.NodeTransformer):
689 657 """Wraps all integers in a call to Integer()"""
690 658 def visit_Num(self, node):
691 659 if isinstance(node.n, int):
692 660 return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
693 661 args=[node], keywords=[])
694 662 return node
695 663
696 664 class TestAstTransform2(unittest.TestCase):
697 665 def setUp(self):
698 666 self.intwrapper = IntegerWrapper()
699 667 ip.ast_transformers.append(self.intwrapper)
700 668
701 669 self.calls = []
702 670 def Integer(*args):
703 671 self.calls.append(args)
704 672 return args
705 673 ip.push({"Integer": Integer})
706 674
707 675 def tearDown(self):
708 676 ip.ast_transformers.remove(self.intwrapper)
709 677 del ip.user_ns['Integer']
710 678
711 679 def test_run_cell(self):
712 680 ip.run_cell("n = 2")
713 681 self.assertEqual(self.calls, [(2,)])
714 682
715 683 # This shouldn't throw an error
716 684 ip.run_cell("o = 2.0")
717 685 self.assertEqual(ip.user_ns['o'], 2.0)
718 686
719 687 def test_timeit(self):
720 688 called = set()
721 689 def f(x):
722 690 called.add(x)
723 691 ip.push({'f':f})
724 692
725 693 with tt.AssertPrints("average of "):
726 694 ip.run_line_magic("timeit", "-n1 f(1)")
727 695 self.assertEqual(called, {(1,)})
728 696 called.clear()
729 697
730 698 with tt.AssertPrints("average of "):
731 699 ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
732 700 self.assertEqual(called, {(2,), (3,)})
733 701
734 702 class ErrorTransformer(ast.NodeTransformer):
735 703 """Throws an error when it sees a number."""
736 704 def visit_Num(self, node):
737 705 raise ValueError("test")
738 706
739 707 class TestAstTransformError(unittest.TestCase):
740 708 def test_unregistering(self):
741 709 err_transformer = ErrorTransformer()
742 710 ip.ast_transformers.append(err_transformer)
743 711
744 712 with tt.AssertPrints("unregister", channel='stderr'):
745 713 ip.run_cell("1 + 2")
746 714
747 715 # This should have been removed.
748 716 nt.assert_not_in(err_transformer, ip.ast_transformers)
749 717
750 718
751 719 class StringRejector(ast.NodeTransformer):
752 720 """Throws an InputRejected when it sees a string literal.
753 721
754 722 Used to verify that NodeTransformers can signal that a piece of code should
755 723 not be executed by throwing an InputRejected.
756 724 """
757 725
758 726 def visit_Str(self, node):
759 727 raise InputRejected("test")
760 728
761 729
762 730 class TestAstTransformInputRejection(unittest.TestCase):
763 731
764 732 def setUp(self):
765 733 self.transformer = StringRejector()
766 734 ip.ast_transformers.append(self.transformer)
767 735
768 736 def tearDown(self):
769 737 ip.ast_transformers.remove(self.transformer)
770 738
771 739 def test_input_rejection(self):
772 740 """Check that NodeTransformers can reject input."""
773 741
774 742 expect_exception_tb = tt.AssertPrints("InputRejected: test")
775 743 expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
776 744
777 745 # Run the same check twice to verify that the transformer is not
778 746 # disabled after raising.
779 747 with expect_exception_tb, expect_no_cell_output:
780 748 ip.run_cell("'unsafe'")
781 749
782 750 with expect_exception_tb, expect_no_cell_output:
783 751 res = ip.run_cell("'unsafe'")
784 752
785 753 self.assertIsInstance(res.error_before_exec, InputRejected)
786 754
787 755 def test__IPYTHON__():
788 756 # This shouldn't raise a NameError, that's all
789 757 __IPYTHON__
790 758
791 759
792 760 class DummyRepr(object):
793 761 def __repr__(self):
794 762 return "DummyRepr"
795 763
796 764 def _repr_html_(self):
797 765 return "<b>dummy</b>"
798 766
799 767 def _repr_javascript_(self):
800 768 return "console.log('hi');", {'key': 'value'}
801 769
802 770
803 771 def test_user_variables():
804 772 # enable all formatters
805 773 ip.display_formatter.active_types = ip.display_formatter.format_types
806 774
807 775 ip.user_ns['dummy'] = d = DummyRepr()
808 776 keys = {'dummy', 'doesnotexist'}
809 777 r = ip.user_expressions({ key:key for key in keys})
810 778
811 779 nt.assert_equal(keys, set(r.keys()))
812 780 dummy = r['dummy']
813 781 nt.assert_equal({'status', 'data', 'metadata'}, set(dummy.keys()))
814 782 nt.assert_equal(dummy['status'], 'ok')
815 783 data = dummy['data']
816 784 metadata = dummy['metadata']
817 785 nt.assert_equal(data.get('text/html'), d._repr_html_())
818 786 js, jsmd = d._repr_javascript_()
819 787 nt.assert_equal(data.get('application/javascript'), js)
820 788 nt.assert_equal(metadata.get('application/javascript'), jsmd)
821 789
822 790 dne = r['doesnotexist']
823 791 nt.assert_equal(dne['status'], 'error')
824 792 nt.assert_equal(dne['ename'], 'NameError')
825 793
826 794 # back to text only
827 795 ip.display_formatter.active_types = ['text/plain']
828 796
829 797 def test_user_expression():
830 798 # enable all formatters
831 799 ip.display_formatter.active_types = ip.display_formatter.format_types
832 800 query = {
833 801 'a' : '1 + 2',
834 802 'b' : '1/0',
835 803 }
836 804 r = ip.user_expressions(query)
837 805 import pprint
838 806 pprint.pprint(r)
839 807 nt.assert_equal(set(r.keys()), set(query.keys()))
840 808 a = r['a']
841 809 nt.assert_equal({'status', 'data', 'metadata'}, set(a.keys()))
842 810 nt.assert_equal(a['status'], 'ok')
843 811 data = a['data']
844 812 metadata = a['metadata']
845 813 nt.assert_equal(data.get('text/plain'), '3')
846 814
847 815 b = r['b']
848 816 nt.assert_equal(b['status'], 'error')
849 817 nt.assert_equal(b['ename'], 'ZeroDivisionError')
850 818
851 819 # back to text only
852 820 ip.display_formatter.active_types = ['text/plain']
853 821
854 822
855 823
856 824
857 825
858 826 class TestSyntaxErrorTransformer(unittest.TestCase):
859 827 """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
860 828
861 829 class SyntaxErrorTransformer(InputTransformer):
862 830
863 831 def push(self, line):
864 832 pos = line.find('syntaxerror')
865 833 if pos >= 0:
866 834 e = SyntaxError('input contains "syntaxerror"')
867 835 e.text = line
868 836 e.offset = pos + 1
869 837 raise e
870 838 return line
871 839
872 840 def reset(self):
873 841 pass
874 842
875 843 def setUp(self):
876 844 self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
877 845 ip.input_splitter.python_line_transforms.append(self.transformer)
878 846 ip.input_transformer_manager.python_line_transforms.append(self.transformer)
879 847
880 848 def tearDown(self):
881 849 ip.input_splitter.python_line_transforms.remove(self.transformer)
882 850 ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
883 851
884 852 def test_syntaxerror_input_transformer(self):
885 853 with tt.AssertPrints('1234'):
886 854 ip.run_cell('1234')
887 855 with tt.AssertPrints('SyntaxError: invalid syntax'):
888 856 ip.run_cell('1 2 3') # plain python syntax error
889 857 with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
890 858 ip.run_cell('2345 # syntaxerror') # input transformer syntax error
891 859 with tt.AssertPrints('3456'):
892 860 ip.run_cell('3456')
893 861
894 862
895 863
896 864 def test_warning_suppression():
897 865 ip.run_cell("import warnings")
898 866 try:
899 867 with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
900 868 ip.run_cell("warnings.warn('asdf')")
901 869 # Here's the real test -- if we run that again, we should get the
902 870 # warning again. Traditionally, each warning was only issued once per
903 871 # IPython session (approximately), even if the user typed in new and
904 872 # different code that should have also triggered the warning, leading
905 873 # to much confusion.
906 874 with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
907 875 ip.run_cell("warnings.warn('asdf')")
908 876 finally:
909 877 ip.run_cell("del warnings")
910 878
911 879
912 880 def test_deprecation_warning():
913 881 ip.run_cell("""
914 882 import warnings
915 883 def wrn():
916 884 warnings.warn(
917 885 "I AM A WARNING",
918 886 DeprecationWarning
919 887 )
920 888 """)
921 889 try:
922 890 with tt.AssertPrints("I AM A WARNING", channel="stderr"):
923 891 ip.run_cell("wrn()")
924 892 finally:
925 893 ip.run_cell("del warnings")
926 894 ip.run_cell("del wrn")
927 895
928 896
929 897 class TestImportNoDeprecate(tt.TempFileMixin):
930 898
931 899 def setup(self):
932 900 """Make a valid python temp file."""
933 901 self.mktmp("""
934 902 import warnings
935 903 def wrn():
936 904 warnings.warn(
937 905 "I AM A WARNING",
938 906 DeprecationWarning
939 907 )
940 908 """)
941 909
942 910 def test_no_dep(self):
943 911 """
944 912 No deprecation warning should be raised from imported functions
945 913 """
946 914 ip.run_cell("from {} import wrn".format(self.fname))
947 915
948 916 with tt.AssertNotPrints("I AM A WARNING"):
949 917 ip.run_cell("wrn()")
950 918 ip.run_cell("del wrn")
@@ -1,1011 +1,988 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for various magic functions.
3 3
4 4 Needs to be run by nose (to make ipython session available).
5 5 """
6 6 from __future__ import absolute_import
7 7
8 8 import io
9 9 import os
10 10 import sys
11 11 import warnings
12 12 from unittest import TestCase
13 13
14 14 try:
15 15 from importlib import invalidate_caches # Required from Python 3.3
16 16 except ImportError:
17 17 def invalidate_caches():
18 18 pass
19 19
20 20 import nose.tools as nt
21 21
22 22 from IPython import get_ipython
23 23 from IPython.core import magic
24 24 from IPython.core.error import UsageError
25 25 from IPython.core.magic import (Magics, magics_class, line_magic,
26 26 cell_magic,
27 27 register_line_magic, register_cell_magic)
28 28 from IPython.core.magics import execution, script, code
29 29 from IPython.testing import decorators as dec
30 30 from IPython.testing import tools as tt
31 31 from IPython.utils import py3compat
32 32 from IPython.utils.io import capture_output
33 33 from IPython.utils.tempdir import TemporaryDirectory
34 34 from IPython.utils.process import find_cmd
35 35
36 36 if py3compat.PY3:
37 37 from io import StringIO
38 38 else:
39 39 from StringIO import StringIO
40 40
41 41
42 42 _ip = get_ipython()
43 43
44 44 @magic.magics_class
45 45 class DummyMagics(magic.Magics): pass
46 46
47 47 def test_extract_code_ranges():
48 48 instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
49 49 expected = [(0, 1),
50 50 (2, 3),
51 51 (4, 6),
52 52 (6, 9),
53 53 (9, 14),
54 54 (16, None),
55 55 (None, 9),
56 56 (9, None),
57 57 (None, 13),
58 58 (None, None)]
59 59 actual = list(code.extract_code_ranges(instr))
60 60 nt.assert_equal(actual, expected)
61 61
62 62 def test_extract_symbols():
63 63 source = """import foo\na = 10\ndef b():\n return 42\n\n\nclass A: pass\n\n\n"""
64 64 symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
65 65 expected = [([], ['a']),
66 66 (["def b():\n return 42\n"], []),
67 67 (["class A: pass\n"], []),
68 68 (["class A: pass\n", "def b():\n return 42\n"], []),
69 69 (["class A: pass\n"], ['a']),
70 70 ([], ['z'])]
71 71 for symbols, exp in zip(symbols_args, expected):
72 72 nt.assert_equal(code.extract_symbols(source, symbols), exp)
73 73
74 74
75 75 def test_extract_symbols_raises_exception_with_non_python_code():
76 76 source = ("=begin A Ruby program :)=end\n"
77 77 "def hello\n"
78 78 "puts 'Hello world'\n"
79 79 "end")
80 80 with nt.assert_raises(SyntaxError):
81 81 code.extract_symbols(source, "hello")
82 82
83 83 def test_config():
84 84 """ test that config magic does not raise
85 85 can happen if Configurable init is moved too early into
86 86 Magics.__init__ as then a Config object will be registerd as a
87 87 magic.
88 88 """
89 89 ## should not raise.
90 90 _ip.magic('config')
91 91
92 92 def test_rehashx():
93 93 # clear up everything
94 94 _ip.alias_manager.clear_aliases()
95 95 del _ip.db['syscmdlist']
96 96
97 97 _ip.magic('rehashx')
98 98 # Practically ALL ipython development systems will have more than 10 aliases
99 99
100 100 nt.assert_true(len(_ip.alias_manager.aliases) > 10)
101 101 for name, cmd in _ip.alias_manager.aliases:
102 102 # we must strip dots from alias names
103 103 nt.assert_not_in('.', name)
104 104
105 105 # rehashx must fill up syscmdlist
106 106 scoms = _ip.db['syscmdlist']
107 107 nt.assert_true(len(scoms) > 10)
108 108
109 109
110 110 def test_magic_parse_options():
111 111 """Test that we don't mangle paths when parsing magic options."""
112 112 ip = get_ipython()
113 113 path = 'c:\\x'
114 114 m = DummyMagics(ip)
115 115 opts = m.parse_options('-f %s' % path,'f:')[0]
116 116 # argv splitting is os-dependent
117 117 if os.name == 'posix':
118 118 expected = 'c:x'
119 119 else:
120 120 expected = path
121 121 nt.assert_equal(opts['f'], expected)
122 122
123 123 def test_magic_parse_long_options():
124 124 """Magic.parse_options can handle --foo=bar long options"""
125 125 ip = get_ipython()
126 126 m = DummyMagics(ip)
127 127 opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
128 128 nt.assert_in('foo', opts)
129 129 nt.assert_in('bar', opts)
130 130 nt.assert_equal(opts['bar'], "bubble")
131 131
132 132
133 133 @dec.skip_without('sqlite3')
134 134 def doctest_hist_f():
135 135 """Test %hist -f with temporary filename.
136 136
137 137 In [9]: import tempfile
138 138
139 139 In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
140 140
141 141 In [11]: %hist -nl -f $tfile 3
142 142
143 143 In [13]: import os; os.unlink(tfile)
144 144 """
145 145
146 146
147 147 @dec.skip_without('sqlite3')
148 148 def doctest_hist_r():
149 149 """Test %hist -r
150 150
151 151 XXX - This test is not recording the output correctly. For some reason, in
152 152 testing mode the raw history isn't getting populated. No idea why.
153 153 Disabling the output checking for now, though at least we do run it.
154 154
155 155 In [1]: 'hist' in _ip.lsmagic()
156 156 Out[1]: True
157 157
158 158 In [2]: x=1
159 159
160 160 In [3]: %hist -rl 2
161 161 x=1 # random
162 162 %hist -r 2
163 163 """
164 164
165 165
166 166 @dec.skip_without('sqlite3')
167 167 def doctest_hist_op():
168 168 """Test %hist -op
169 169
170 170 In [1]: class b(float):
171 171 ...: pass
172 172 ...:
173 173
174 174 In [2]: class s(object):
175 175 ...: def __str__(self):
176 176 ...: return 's'
177 177 ...:
178 178
179 179 In [3]:
180 180
181 181 In [4]: class r(b):
182 182 ...: def __repr__(self):
183 183 ...: return 'r'
184 184 ...:
185 185
186 186 In [5]: class sr(s,r): pass
187 187 ...:
188 188
189 189 In [6]:
190 190
191 191 In [7]: bb=b()
192 192
193 193 In [8]: ss=s()
194 194
195 195 In [9]: rr=r()
196 196
197 197 In [10]: ssrr=sr()
198 198
199 199 In [11]: 4.5
200 200 Out[11]: 4.5
201 201
202 202 In [12]: str(ss)
203 203 Out[12]: 's'
204 204
205 205 In [13]:
206 206
207 207 In [14]: %hist -op
208 208 >>> class b:
209 209 ... pass
210 210 ...
211 211 >>> class s(b):
212 212 ... def __str__(self):
213 213 ... return 's'
214 214 ...
215 215 >>>
216 216 >>> class r(b):
217 217 ... def __repr__(self):
218 218 ... return 'r'
219 219 ...
220 220 >>> class sr(s,r): pass
221 221 >>>
222 222 >>> bb=b()
223 223 >>> ss=s()
224 224 >>> rr=r()
225 225 >>> ssrr=sr()
226 226 >>> 4.5
227 227 4.5
228 228 >>> str(ss)
229 229 's'
230 230 >>>
231 231 """
232 232
233 233 def test_hist_pof():
234 234 ip = get_ipython()
235 235 ip.run_cell(u"1+2", store_history=True)
236 236 #raise Exception(ip.history_manager.session_number)
237 237 #raise Exception(list(ip.history_manager._get_range_session()))
238 238 with TemporaryDirectory() as td:
239 239 tf = os.path.join(td, 'hist.py')
240 240 ip.run_line_magic('history', '-pof %s' % tf)
241 241 assert os.path.isfile(tf)
242 242
243 243
244 244 @dec.skip_without('sqlite3')
245 245 def test_macro():
246 246 ip = get_ipython()
247 247 ip.history_manager.reset() # Clear any existing history.
248 248 cmds = ["a=1", "def b():\n return a**2", "print(a,b())"]
249 249 for i, cmd in enumerate(cmds, start=1):
250 250 ip.history_manager.store_inputs(i, cmd)
251 251 ip.magic("macro test 1-3")
252 252 nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
253 253
254 254 # List macros
255 255 nt.assert_in("test", ip.magic("macro"))
256 256
257 257
258 258 @dec.skip_without('sqlite3')
259 259 def test_macro_run():
260 260 """Test that we can run a multi-line macro successfully."""
261 261 ip = get_ipython()
262 262 ip.history_manager.reset()
263 263 cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
264 264 "%macro test 2-3"]
265 265 for cmd in cmds:
266 266 ip.run_cell(cmd, store_history=True)
267 267 nt.assert_equal(ip.user_ns["test"].value,
268 268 py3compat.doctest_refactor_print("a+=1\nprint a\n"))
269 269 with tt.AssertPrints("12"):
270 270 ip.run_cell("test")
271 271 with tt.AssertPrints("13"):
272 272 ip.run_cell("test")
273 273
274 274
275 275 def test_magic_magic():
276 276 """Test %magic"""
277 277 ip = get_ipython()
278 278 with capture_output() as captured:
279 279 ip.magic("magic")
280 280
281 281 stdout = captured.stdout
282 282 nt.assert_in('%magic', stdout)
283 283 nt.assert_in('IPython', stdout)
284 284 nt.assert_in('Available', stdout)
285 285
286 286
287 287 @dec.skipif_not_numpy
288 288 def test_numpy_reset_array_undec():
289 289 "Test '%reset array' functionality"
290 290 _ip.ex('import numpy as np')
291 291 _ip.ex('a = np.empty(2)')
292 292 nt.assert_in('a', _ip.user_ns)
293 293 _ip.magic('reset -f array')
294 294 nt.assert_not_in('a', _ip.user_ns)
295 295
296 296 def test_reset_out():
297 297 "Test '%reset out' magic"
298 298 _ip.run_cell("parrot = 'dead'", store_history=True)
299 299 # test '%reset -f out', make an Out prompt
300 300 _ip.run_cell("parrot", store_history=True)
301 301 nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
302 302 _ip.magic('reset -f out')
303 303 nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
304 304 nt.assert_equal(len(_ip.user_ns['Out']), 0)
305 305
306 306 def test_reset_in():
307 307 "Test '%reset in' magic"
308 308 # test '%reset -f in'
309 309 _ip.run_cell("parrot", store_history=True)
310 310 nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
311 311 _ip.magic('%reset -f in')
312 312 nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
313 313 nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
314 314
315 315 def test_reset_dhist():
316 316 "Test '%reset dhist' magic"
317 317 _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
318 318 _ip.magic('cd ' + os.path.dirname(nt.__file__))
319 319 _ip.magic('cd -')
320 320 nt.assert_true(len(_ip.user_ns['_dh']) > 0)
321 321 _ip.magic('reset -f dhist')
322 322 nt.assert_equal(len(_ip.user_ns['_dh']), 0)
323 323 _ip.run_cell("_dh = [d for d in tmp]") #restore
324 324
325 325 def test_reset_in_length():
326 326 "Test that '%reset in' preserves In[] length"
327 327 _ip.run_cell("print 'foo'")
328 328 _ip.run_cell("reset -f in")
329 329 nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
330 330
331 331 def test_tb_syntaxerror():
332 332 """test %tb after a SyntaxError"""
333 333 ip = get_ipython()
334 334 ip.run_cell("for")
335 335
336 336 # trap and validate stdout
337 337 save_stdout = sys.stdout
338 338 try:
339 339 sys.stdout = StringIO()
340 340 ip.run_cell("%tb")
341 341 out = sys.stdout.getvalue()
342 342 finally:
343 343 sys.stdout = save_stdout
344 344 # trim output, and only check the last line
345 345 last_line = out.rstrip().splitlines()[-1].strip()
346 346 nt.assert_equal(last_line, "SyntaxError: invalid syntax")
347 347
348 348
349 349 def test_time():
350 350 ip = get_ipython()
351 351
352 352 with tt.AssertPrints("Wall time: "):
353 353 ip.run_cell("%time None")
354 354
355 355 ip.run_cell("def f(kmjy):\n"
356 356 " %time print (2*kmjy)")
357 357
358 358 with tt.AssertPrints("Wall time: "):
359 359 with tt.AssertPrints("hihi", suppress=False):
360 360 ip.run_cell("f('hi')")
361 361
362 362
363 363 @dec.skip_win32
364 364 def test_time2():
365 365 ip = get_ipython()
366 366
367 367 with tt.AssertPrints("CPU times: user "):
368 368 ip.run_cell("%time None")
369 369
370 370 def test_time3():
371 371 """Erroneous magic function calls, issue gh-3334"""
372 372 ip = get_ipython()
373 373 ip.user_ns.pop('run', None)
374 374
375 375 with tt.AssertNotPrints("not found", channel='stderr'):
376 376 ip.run_cell("%%time\n"
377 377 "run = 0\n"
378 378 "run += 1")
379 379
380 @dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
381 def test_time_futures():
382 "Test %time with __future__ environments"
383 ip = get_ipython()
384 ip.autocall = 0
385 ip.run_cell("from __future__ import division")
386 with tt.AssertPrints('0.25'):
387 ip.run_line_magic('time', 'print(1/4)')
388 ip.compile.reset_compiler_flags()
389 with tt.AssertNotPrints('0.25'):
390 ip.run_line_magic('time', 'print(1/4)')
391
392 380 def test_doctest_mode():
393 381 "Toggle doctest_mode twice, it should be a no-op and run without error"
394 382 _ip.magic('doctest_mode')
395 383 _ip.magic('doctest_mode')
396 384
397 385
398 386 def test_parse_options():
399 387 """Tests for basic options parsing in magics."""
400 388 # These are only the most minimal of tests, more should be added later. At
401 389 # the very least we check that basic text/unicode calls work OK.
402 390 m = DummyMagics(_ip)
403 391 nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
404 392 nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
405 393
406 394
407 395 def test_dirops():
408 396 """Test various directory handling operations."""
409 397 # curpath = lambda :os.path.splitdrive(py3compat.getcwd())[1].replace('\\','/')
410 398 curpath = py3compat.getcwd
411 399 startdir = py3compat.getcwd()
412 400 ipdir = os.path.realpath(_ip.ipython_dir)
413 401 try:
414 402 _ip.magic('cd "%s"' % ipdir)
415 403 nt.assert_equal(curpath(), ipdir)
416 404 _ip.magic('cd -')
417 405 nt.assert_equal(curpath(), startdir)
418 406 _ip.magic('pushd "%s"' % ipdir)
419 407 nt.assert_equal(curpath(), ipdir)
420 408 _ip.magic('popd')
421 409 nt.assert_equal(curpath(), startdir)
422 410 finally:
423 411 os.chdir(startdir)
424 412
425 413
426 414 def test_xmode():
427 415 # Calling xmode three times should be a no-op
428 416 xmode = _ip.InteractiveTB.mode
429 417 for i in range(3):
430 418 _ip.magic("xmode")
431 419 nt.assert_equal(_ip.InteractiveTB.mode, xmode)
432 420
433 421 def test_reset_hard():
434 422 monitor = []
435 423 class A(object):
436 424 def __del__(self):
437 425 monitor.append(1)
438 426 def __repr__(self):
439 427 return "<A instance>"
440 428
441 429 _ip.user_ns["a"] = A()
442 430 _ip.run_cell("a")
443 431
444 432 nt.assert_equal(monitor, [])
445 433 _ip.magic("reset -f")
446 434 nt.assert_equal(monitor, [1])
447 435
448 436 class TestXdel(tt.TempFileMixin):
449 437 def test_xdel(self):
450 438 """Test that references from %run are cleared by xdel."""
451 439 src = ("class A(object):\n"
452 440 " monitor = []\n"
453 441 " def __del__(self):\n"
454 442 " self.monitor.append(1)\n"
455 443 "a = A()\n")
456 444 self.mktmp(src)
457 445 # %run creates some hidden references...
458 446 _ip.magic("run %s" % self.fname)
459 447 # ... as does the displayhook.
460 448 _ip.run_cell("a")
461 449
462 450 monitor = _ip.user_ns["A"].monitor
463 451 nt.assert_equal(monitor, [])
464 452
465 453 _ip.magic("xdel a")
466 454
467 455 # Check that a's __del__ method has been called.
468 456 nt.assert_equal(monitor, [1])
469 457
470 458 def doctest_who():
471 459 """doctest for %who
472 460
473 461 In [1]: %reset -f
474 462
475 463 In [2]: alpha = 123
476 464
477 465 In [3]: beta = 'beta'
478 466
479 467 In [4]: %who int
480 468 alpha
481 469
482 470 In [5]: %who str
483 471 beta
484 472
485 473 In [6]: %whos
486 474 Variable Type Data/Info
487 475 ----------------------------
488 476 alpha int 123
489 477 beta str beta
490 478
491 479 In [7]: %who_ls
492 480 Out[7]: ['alpha', 'beta']
493 481 """
494 482
495 483 def test_whos():
496 484 """Check that whos is protected against objects where repr() fails."""
497 485 class A(object):
498 486 def __repr__(self):
499 487 raise Exception()
500 488 _ip.user_ns['a'] = A()
501 489 _ip.magic("whos")
502 490
503 491 @py3compat.u_format
504 492 def doctest_precision():
505 493 """doctest for %precision
506 494
507 495 In [1]: f = get_ipython().display_formatter.formatters['text/plain']
508 496
509 497 In [2]: %precision 5
510 498 Out[2]: {u}'%.5f'
511 499
512 500 In [3]: f.float_format
513 501 Out[3]: {u}'%.5f'
514 502
515 503 In [4]: %precision %e
516 504 Out[4]: {u}'%e'
517 505
518 506 In [5]: f(3.1415927)
519 507 Out[5]: {u}'3.141593e+00'
520 508 """
521 509
522 510 def test_psearch():
523 511 with tt.AssertPrints("dict.fromkeys"):
524 512 _ip.run_cell("dict.fr*?")
525 513
526 514 def test_timeit_shlex():
527 515 """test shlex issues with timeit (#1109)"""
528 516 _ip.ex("def f(*a,**kw): pass")
529 517 _ip.magic('timeit -n1 "this is a bug".count(" ")')
530 518 _ip.magic('timeit -r1 -n1 f(" ", 1)')
531 519 _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
532 520 _ip.magic('timeit -r1 -n1 ("a " + "b")')
533 521 _ip.magic('timeit -r1 -n1 f("a " + "b")')
534 522 _ip.magic('timeit -r1 -n1 f("a " + "b ")')
535 523
536 524
537 525 def test_timeit_arguments():
538 526 "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
539 527 _ip.magic("timeit ('#')")
540 528
541 529
542 530 def test_timeit_special_syntax():
543 531 "Test %%timeit with IPython special syntax"
544 532 @register_line_magic
545 533 def lmagic(line):
546 534 ip = get_ipython()
547 535 ip.user_ns['lmagic_out'] = line
548 536
549 537 # line mode test
550 538 _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
551 539 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
552 540 # cell mode test
553 541 _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
554 542 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
555 543
556 544 def test_timeit_return():
557 545 """
558 546 test wether timeit -o return object
559 547 """
560 548
561 549 res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
562 550 assert(res is not None)
563 551
564 552 def test_timeit_quiet():
565 553 """
566 554 test quiet option of timeit magic
567 555 """
568 556 with tt.AssertNotPrints("loops"):
569 557 _ip.run_cell("%timeit -n1 -r1 -q 1")
570 558
571 559 def test_timeit_return_quiet():
572 560 with tt.AssertNotPrints("loops"):
573 561 res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
574 562 assert (res is not None)
575 563
576 @dec.skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
577 def test_timeit_futures():
578 "Test %timeit with __future__ environments"
579 ip = get_ipython()
580 ip.run_cell("from __future__ import division")
581 with tt.AssertPrints('0.25'):
582 ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
583 ip.compile.reset_compiler_flags()
584 with tt.AssertNotPrints('0.25'):
585 ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
586
587 564 @dec.skipif(execution.profile is None)
588 565 def test_prun_special_syntax():
589 566 "Test %%prun with IPython special syntax"
590 567 @register_line_magic
591 568 def lmagic(line):
592 569 ip = get_ipython()
593 570 ip.user_ns['lmagic_out'] = line
594 571
595 572 # line mode test
596 573 _ip.run_line_magic('prun', '-q %lmagic my line')
597 574 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
598 575 # cell mode test
599 576 _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
600 577 nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
601 578
602 579 @dec.skipif(execution.profile is None)
603 580 def test_prun_quotes():
604 581 "Test that prun does not clobber string escapes (GH #1302)"
605 582 _ip.magic(r"prun -q x = '\t'")
606 583 nt.assert_equal(_ip.user_ns['x'], '\t')
607 584
608 585 def test_extension():
609 586 # Debugging information for failures of this test
610 587 print('sys.path:')
611 588 for p in sys.path:
612 589 print(' ', p)
613 590 print('CWD', os.getcwd())
614 591
615 592 nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
616 593 daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
617 594 sys.path.insert(0, daft_path)
618 595 try:
619 596 _ip.user_ns.pop('arq', None)
620 597 invalidate_caches() # Clear import caches
621 598 _ip.magic("load_ext daft_extension")
622 599 nt.assert_equal(_ip.user_ns['arq'], 185)
623 600 _ip.magic("unload_ext daft_extension")
624 601 assert 'arq' not in _ip.user_ns
625 602 finally:
626 603 sys.path.remove(daft_path)
627 604
628 605
629 606 def test_notebook_export_json():
630 607 _ip = get_ipython()
631 608 _ip.history_manager.reset() # Clear any existing history.
632 609 cmds = [u"a=1", u"def b():\n return a**2", u"print('noΓ«l, Γ©tΓ©', b())"]
633 610 for i, cmd in enumerate(cmds, start=1):
634 611 _ip.history_manager.store_inputs(i, cmd)
635 612 with TemporaryDirectory() as td:
636 613 outfile = os.path.join(td, "nb.ipynb")
637 614 _ip.magic("notebook -e %s" % outfile)
638 615
639 616
640 617 class TestEnv(TestCase):
641 618
642 619 def test_env(self):
643 620 env = _ip.magic("env")
644 621 self.assertTrue(isinstance(env, dict))
645 622
646 623 def test_env_get_set_simple(self):
647 624 env = _ip.magic("env var val1")
648 625 self.assertEqual(env, None)
649 626 self.assertEqual(os.environ['var'], 'val1')
650 627 self.assertEqual(_ip.magic("env var"), 'val1')
651 628 env = _ip.magic("env var=val2")
652 629 self.assertEqual(env, None)
653 630 self.assertEqual(os.environ['var'], 'val2')
654 631
655 632 def test_env_get_set_complex(self):
656 633 env = _ip.magic("env var 'val1 '' 'val2")
657 634 self.assertEqual(env, None)
658 635 self.assertEqual(os.environ['var'], "'val1 '' 'val2")
659 636 self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
660 637 env = _ip.magic('env var=val2 val3="val4')
661 638 self.assertEqual(env, None)
662 639 self.assertEqual(os.environ['var'], 'val2 val3="val4')
663 640
664 641 def test_env_set_bad_input(self):
665 642 self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
666 643
667 644 def test_env_set_whitespace(self):
668 645 self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
669 646
670 647
671 648 class CellMagicTestCase(TestCase):
672 649
673 650 def check_ident(self, magic):
674 651 # Manually called, we get the result
675 652 out = _ip.run_cell_magic(magic, 'a', 'b')
676 653 nt.assert_equal(out, ('a','b'))
677 654 # Via run_cell, it goes into the user's namespace via displayhook
678 655 _ip.run_cell('%%' + magic +' c\nd')
679 656 nt.assert_equal(_ip.user_ns['_'], ('c','d'))
680 657
681 658 def test_cell_magic_func_deco(self):
682 659 "Cell magic using simple decorator"
683 660 @register_cell_magic
684 661 def cellm(line, cell):
685 662 return line, cell
686 663
687 664 self.check_ident('cellm')
688 665
689 666 def test_cell_magic_reg(self):
690 667 "Cell magic manually registered"
691 668 def cellm(line, cell):
692 669 return line, cell
693 670
694 671 _ip.register_magic_function(cellm, 'cell', 'cellm2')
695 672 self.check_ident('cellm2')
696 673
697 674 def test_cell_magic_class(self):
698 675 "Cell magics declared via a class"
699 676 @magics_class
700 677 class MyMagics(Magics):
701 678
702 679 @cell_magic
703 680 def cellm3(self, line, cell):
704 681 return line, cell
705 682
706 683 _ip.register_magics(MyMagics)
707 684 self.check_ident('cellm3')
708 685
709 686 def test_cell_magic_class2(self):
710 687 "Cell magics declared via a class, #2"
711 688 @magics_class
712 689 class MyMagics2(Magics):
713 690
714 691 @cell_magic('cellm4')
715 692 def cellm33(self, line, cell):
716 693 return line, cell
717 694
718 695 _ip.register_magics(MyMagics2)
719 696 self.check_ident('cellm4')
720 697 # Check that nothing is registered as 'cellm33'
721 698 c33 = _ip.find_cell_magic('cellm33')
722 699 nt.assert_equal(c33, None)
723 700
724 701 def test_file():
725 702 """Basic %%file"""
726 703 ip = get_ipython()
727 704 with TemporaryDirectory() as td:
728 705 fname = os.path.join(td, 'file1')
729 706 ip.run_cell_magic("file", fname, u'\n'.join([
730 707 'line1',
731 708 'line2',
732 709 ]))
733 710 with open(fname) as f:
734 711 s = f.read()
735 712 nt.assert_in('line1\n', s)
736 713 nt.assert_in('line2', s)
737 714
738 715 def test_file_var_expand():
739 716 """%%file $filename"""
740 717 ip = get_ipython()
741 718 with TemporaryDirectory() as td:
742 719 fname = os.path.join(td, 'file1')
743 720 ip.user_ns['filename'] = fname
744 721 ip.run_cell_magic("file", '$filename', u'\n'.join([
745 722 'line1',
746 723 'line2',
747 724 ]))
748 725 with open(fname) as f:
749 726 s = f.read()
750 727 nt.assert_in('line1\n', s)
751 728 nt.assert_in('line2', s)
752 729
753 730 def test_file_unicode():
754 731 """%%file with unicode cell"""
755 732 ip = get_ipython()
756 733 with TemporaryDirectory() as td:
757 734 fname = os.path.join(td, 'file1')
758 735 ip.run_cell_magic("file", fname, u'\n'.join([
759 736 u'linΓ©1',
760 737 u'linΓ©2',
761 738 ]))
762 739 with io.open(fname, encoding='utf-8') as f:
763 740 s = f.read()
764 741 nt.assert_in(u'linΓ©1\n', s)
765 742 nt.assert_in(u'linΓ©2', s)
766 743
767 744 def test_file_amend():
768 745 """%%file -a amends files"""
769 746 ip = get_ipython()
770 747 with TemporaryDirectory() as td:
771 748 fname = os.path.join(td, 'file2')
772 749 ip.run_cell_magic("file", fname, u'\n'.join([
773 750 'line1',
774 751 'line2',
775 752 ]))
776 753 ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([
777 754 'line3',
778 755 'line4',
779 756 ]))
780 757 with open(fname) as f:
781 758 s = f.read()
782 759 nt.assert_in('line1\n', s)
783 760 nt.assert_in('line3\n', s)
784 761
785 762
786 763 def test_script_config():
787 764 ip = get_ipython()
788 765 ip.config.ScriptMagics.script_magics = ['whoda']
789 766 sm = script.ScriptMagics(shell=ip)
790 767 nt.assert_in('whoda', sm.magics['cell'])
791 768
792 769 @dec.skip_win32
793 770 def test_script_out():
794 771 ip = get_ipython()
795 772 ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
796 773 nt.assert_equal(ip.user_ns['output'], 'hi\n')
797 774
798 775 @dec.skip_win32
799 776 def test_script_err():
800 777 ip = get_ipython()
801 778 ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
802 779 nt.assert_equal(ip.user_ns['error'], 'hello\n')
803 780
804 781 @dec.skip_win32
805 782 def test_script_out_err():
806 783 ip = get_ipython()
807 784 ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
808 785 nt.assert_equal(ip.user_ns['output'], 'hi\n')
809 786 nt.assert_equal(ip.user_ns['error'], 'hello\n')
810 787
811 788 @dec.skip_win32
812 789 def test_script_bg_out():
813 790 ip = get_ipython()
814 791 ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
815 792 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
816 793
817 794 @dec.skip_win32
818 795 def test_script_bg_err():
819 796 ip = get_ipython()
820 797 ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
821 798 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
822 799
823 800 @dec.skip_win32
824 801 def test_script_bg_out_err():
825 802 ip = get_ipython()
826 803 ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
827 804 nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
828 805 nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
829 806
830 807 def test_script_defaults():
831 808 ip = get_ipython()
832 809 for cmd in ['sh', 'bash', 'perl', 'ruby']:
833 810 try:
834 811 find_cmd(cmd)
835 812 except Exception:
836 813 pass
837 814 else:
838 815 nt.assert_in(cmd, ip.magics_manager.magics['cell'])
839 816
840 817
841 818 @magics_class
842 819 class FooFoo(Magics):
843 820 """class with both %foo and %%foo magics"""
844 821 @line_magic('foo')
845 822 def line_foo(self, line):
846 823 "I am line foo"
847 824 pass
848 825
849 826 @cell_magic("foo")
850 827 def cell_foo(self, line, cell):
851 828 "I am cell foo, not line foo"
852 829 pass
853 830
854 831 def test_line_cell_info():
855 832 """%%foo and %foo magics are distinguishable to inspect"""
856 833 ip = get_ipython()
857 834 ip.magics_manager.register(FooFoo)
858 835 oinfo = ip.object_inspect('foo')
859 836 nt.assert_true(oinfo['found'])
860 837 nt.assert_true(oinfo['ismagic'])
861 838
862 839 oinfo = ip.object_inspect('%%foo')
863 840 nt.assert_true(oinfo['found'])
864 841 nt.assert_true(oinfo['ismagic'])
865 842 nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
866 843
867 844 oinfo = ip.object_inspect('%foo')
868 845 nt.assert_true(oinfo['found'])
869 846 nt.assert_true(oinfo['ismagic'])
870 847 nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
871 848
872 849 def test_multiple_magics():
873 850 ip = get_ipython()
874 851 foo1 = FooFoo(ip)
875 852 foo2 = FooFoo(ip)
876 853 mm = ip.magics_manager
877 854 mm.register(foo1)
878 855 nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
879 856 mm.register(foo2)
880 857 nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
881 858
882 859 def test_alias_magic():
883 860 """Test %alias_magic."""
884 861 ip = get_ipython()
885 862 mm = ip.magics_manager
886 863
887 864 # Basic operation: both cell and line magics are created, if possible.
888 865 ip.run_line_magic('alias_magic', 'timeit_alias timeit')
889 866 nt.assert_in('timeit_alias', mm.magics['line'])
890 867 nt.assert_in('timeit_alias', mm.magics['cell'])
891 868
892 869 # --cell is specified, line magic not created.
893 870 ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
894 871 nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
895 872 nt.assert_in('timeit_cell_alias', mm.magics['cell'])
896 873
897 874 # Test that line alias is created successfully.
898 875 ip.run_line_magic('alias_magic', '--line env_alias env')
899 876 nt.assert_equal(ip.run_line_magic('env', ''),
900 877 ip.run_line_magic('env_alias', ''))
901 878
902 879 def test_save():
903 880 """Test %save."""
904 881 ip = get_ipython()
905 882 ip.history_manager.reset() # Clear any existing history.
906 883 cmds = [u"a=1", u"def b():\n return a**2", u"print(a, b())"]
907 884 for i, cmd in enumerate(cmds, start=1):
908 885 ip.history_manager.store_inputs(i, cmd)
909 886 with TemporaryDirectory() as tmpdir:
910 887 file = os.path.join(tmpdir, "testsave.py")
911 888 ip.run_line_magic("save", "%s 1-10" % file)
912 889 with open(file) as f:
913 890 content = f.read()
914 891 nt.assert_equal(content.count(cmds[0]), 1)
915 892 nt.assert_in('coding: utf-8', content)
916 893 ip.run_line_magic("save", "-a %s 1-10" % file)
917 894 with open(file) as f:
918 895 content = f.read()
919 896 nt.assert_equal(content.count(cmds[0]), 2)
920 897 nt.assert_in('coding: utf-8', content)
921 898
922 899
923 900 def test_store():
924 901 """Test %store."""
925 902 ip = get_ipython()
926 903 ip.run_line_magic('load_ext', 'storemagic')
927 904
928 905 # make sure the storage is empty
929 906 ip.run_line_magic('store', '-z')
930 907 ip.user_ns['var'] = 42
931 908 ip.run_line_magic('store', 'var')
932 909 ip.user_ns['var'] = 39
933 910 ip.run_line_magic('store', '-r')
934 911 nt.assert_equal(ip.user_ns['var'], 42)
935 912
936 913 ip.run_line_magic('store', '-d var')
937 914 ip.user_ns['var'] = 39
938 915 ip.run_line_magic('store' , '-r')
939 916 nt.assert_equal(ip.user_ns['var'], 39)
940 917
941 918
942 919 def _run_edit_test(arg_s, exp_filename=None,
943 920 exp_lineno=-1,
944 921 exp_contents=None,
945 922 exp_is_temp=None):
946 923 ip = get_ipython()
947 924 M = code.CodeMagics(ip)
948 925 last_call = ['','']
949 926 opts,args = M.parse_options(arg_s,'prxn:')
950 927 filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
951 928
952 929 if exp_filename is not None:
953 930 nt.assert_equal(exp_filename, filename)
954 931 if exp_contents is not None:
955 932 with io.open(filename, 'r', encoding='utf-8') as f:
956 933 contents = f.read()
957 934 nt.assert_equal(exp_contents, contents)
958 935 if exp_lineno != -1:
959 936 nt.assert_equal(exp_lineno, lineno)
960 937 if exp_is_temp is not None:
961 938 nt.assert_equal(exp_is_temp, is_temp)
962 939
963 940
964 941 def test_edit_interactive():
965 942 """%edit on interactively defined objects"""
966 943 ip = get_ipython()
967 944 n = ip.execution_count
968 945 ip.run_cell(u"def foo(): return 1", store_history=True)
969 946
970 947 try:
971 948 _run_edit_test("foo")
972 949 except code.InteractivelyDefined as e:
973 950 nt.assert_equal(e.index, n)
974 951 else:
975 952 raise AssertionError("Should have raised InteractivelyDefined")
976 953
977 954
978 955 def test_edit_cell():
979 956 """%edit [cell id]"""
980 957 ip = get_ipython()
981 958
982 959 ip.run_cell(u"def foo(): return 1", store_history=True)
983 960
984 961 # test
985 962 _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
986 963
987 964 def test_bookmark():
988 965 ip = get_ipython()
989 966 ip.run_line_magic('bookmark', 'bmname')
990 967 with tt.AssertPrints('bmname'):
991 968 ip.run_line_magic('bookmark', '-l')
992 969 ip.run_line_magic('bookmark', '-d bmname')
993 970
994 971 def test_ls_magic():
995 972 ip = get_ipython()
996 973 json_formatter = ip.display_formatter.formatters['application/json']
997 974 json_formatter.enabled = True
998 975 lsmagic = ip.magic('lsmagic')
999 976 with warnings.catch_warnings(record=True) as w:
1000 977 j = json_formatter(lsmagic)
1001 978 nt.assert_equal(sorted(j), ['cell', 'line'])
1002 979 nt.assert_equal(w, []) # no warnings
1003 980
1004 981 def test_strip_initial_indent():
1005 982 def sii(s):
1006 983 lines = s.splitlines()
1007 984 return '\n'.join(code.strip_initial_indent(lines))
1008 985
1009 986 nt.assert_equal(sii(" a = 1\nb = 2"), "a = 1\nb = 2")
1010 987 nt.assert_equal(sii(" a\n b\nc"), "a\n b\nc")
1011 988 nt.assert_equal(sii("a\n b"), "a\n b")
@@ -1,210 +1,203 b''
1 1 """Tests for various magic functions specific to the terminal frontend.
2 2
3 3 Needs to be run by nose (to make ipython session available).
4 4 """
5 5 from __future__ import absolute_import
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Imports
9 9 #-----------------------------------------------------------------------------
10 10
11 11 import sys
12 12 from unittest import TestCase
13 13
14 14 import nose.tools as nt
15 15
16 16 from IPython.testing import tools as tt
17 17 from IPython.utils.py3compat import PY3
18 18
19 19 if PY3:
20 20 from io import StringIO
21 21 else:
22 22 from StringIO import StringIO
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Globals
26 26 #-----------------------------------------------------------------------------
27 27 ip = get_ipython()
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Test functions begin
31 31 #-----------------------------------------------------------------------------
32 32
33 33 def check_cpaste(code, should_fail=False):
34 34 """Execute code via 'cpaste' and ensure it was executed, unless
35 35 should_fail is set.
36 36 """
37 37 ip.user_ns['code_ran'] = False
38 38
39 39 src = StringIO()
40 40 if not hasattr(src, 'encoding'):
41 41 # IPython expects stdin to have an encoding attribute
42 42 src.encoding = None
43 43 src.write(code)
44 44 src.write('\n--\n')
45 45 src.seek(0)
46 46
47 47 stdin_save = sys.stdin
48 48 sys.stdin = src
49 49
50 50 try:
51 51 context = tt.AssertPrints if should_fail else tt.AssertNotPrints
52 52 with context("Traceback (most recent call last)"):
53 53 ip.magic('cpaste')
54 54
55 55 if not should_fail:
56 56 assert ip.user_ns['code_ran'], "%r failed" % code
57 57 finally:
58 58 sys.stdin = stdin_save
59 59
60 PY31 = sys.version_info[:2] == (3,1)
61
62 60 def test_cpaste():
63 61 """Test cpaste magic"""
64 62
65 63 def runf():
66 64 """Marker function: sets a flag when executed.
67 65 """
68 66 ip.user_ns['code_ran'] = True
69 67 return 'runf' # return string so '+ runf()' doesn't result in success
70 68
71 69 tests = {'pass': ["runf()",
72 70 "In [1]: runf()",
73 71 "In [1]: if 1:\n ...: runf()",
74 72 "> > > runf()",
75 73 ">>> runf()",
76 74 " >>> runf()",
77 75 ],
78 76
79 77 'fail': ["1 + runf()",
78 "++ runf()",
80 79 ]}
81
82 # I don't know why this is failing specifically on Python 3.1. I've
83 # checked it manually interactively, but we don't care enough about 3.1
84 # to spend time fiddling with the tests, so we just skip it.
85 if not PY31:
86 tests['fail'].append("++ runf()")
87 80
88 81 ip.user_ns['runf'] = runf
89 82
90 83 for code in tests['pass']:
91 84 check_cpaste(code)
92 85
93 86 for code in tests['fail']:
94 87 check_cpaste(code, should_fail=True)
95 88
96 89
97 90 class PasteTestCase(TestCase):
98 91 """Multiple tests for clipboard pasting"""
99 92
100 93 def paste(self, txt, flags='-q'):
101 94 """Paste input text, by default in quiet mode"""
102 95 ip.hooks.clipboard_get = lambda : txt
103 96 ip.magic('paste '+flags)
104 97
105 98 def setUp(self):
106 99 # Inject fake clipboard hook but save original so we can restore it later
107 100 self.original_clip = ip.hooks.clipboard_get
108 101
109 102 def tearDown(self):
110 103 # Restore original hook
111 104 ip.hooks.clipboard_get = self.original_clip
112 105
113 106 def test_paste(self):
114 107 ip.user_ns.pop('x', None)
115 108 self.paste('x = 1')
116 109 nt.assert_equal(ip.user_ns['x'], 1)
117 110 ip.user_ns.pop('x')
118 111
119 112 def test_paste_pyprompt(self):
120 113 ip.user_ns.pop('x', None)
121 114 self.paste('>>> x=2')
122 115 nt.assert_equal(ip.user_ns['x'], 2)
123 116 ip.user_ns.pop('x')
124 117
125 118 def test_paste_py_multi(self):
126 119 self.paste("""
127 120 >>> x = [1,2,3]
128 121 >>> y = []
129 122 >>> for i in x:
130 123 ... y.append(i**2)
131 124 ...
132 125 """)
133 126 nt.assert_equal(ip.user_ns['x'], [1,2,3])
134 127 nt.assert_equal(ip.user_ns['y'], [1,4,9])
135 128
136 129 def test_paste_py_multi_r(self):
137 130 "Now, test that self.paste -r works"
138 131 self.test_paste_py_multi()
139 132 nt.assert_equal(ip.user_ns.pop('x'), [1,2,3])
140 133 nt.assert_equal(ip.user_ns.pop('y'), [1,4,9])
141 134 nt.assert_false('x' in ip.user_ns)
142 135 ip.magic('paste -r')
143 136 nt.assert_equal(ip.user_ns['x'], [1,2,3])
144 137 nt.assert_equal(ip.user_ns['y'], [1,4,9])
145 138
146 139 def test_paste_email(self):
147 140 "Test pasting of email-quoted contents"
148 141 self.paste("""\
149 142 >> def foo(x):
150 143 >> return x + 1
151 144 >> xx = foo(1.1)""")
152 145 nt.assert_equal(ip.user_ns['xx'], 2.1)
153 146
154 147 def test_paste_email2(self):
155 148 "Email again; some programs add a space also at each quoting level"
156 149 self.paste("""\
157 150 > > def foo(x):
158 151 > > return x + 1
159 152 > > yy = foo(2.1) """)
160 153 nt.assert_equal(ip.user_ns['yy'], 3.1)
161 154
162 155 def test_paste_email_py(self):
163 156 "Email quoting of interactive input"
164 157 self.paste("""\
165 158 >> >>> def f(x):
166 159 >> ... return x+1
167 160 >> ...
168 161 >> >>> zz = f(2.5) """)
169 162 nt.assert_equal(ip.user_ns['zz'], 3.5)
170 163
171 164 def test_paste_echo(self):
172 165 "Also test self.paste echoing, by temporarily faking the writer"
173 166 w = StringIO()
174 167 writer = ip.write
175 168 ip.write = w.write
176 169 code = """
177 170 a = 100
178 171 b = 200"""
179 172 try:
180 173 self.paste(code,'')
181 174 out = w.getvalue()
182 175 finally:
183 176 ip.write = writer
184 177 nt.assert_equal(ip.user_ns['a'], 100)
185 178 nt.assert_equal(ip.user_ns['b'], 200)
186 179 nt.assert_equal(out, code+"\n## -- End pasted text --\n")
187 180
188 181 def test_paste_leading_commas(self):
189 182 "Test multiline strings with leading commas"
190 183 tm = ip.magics_manager.registry['TerminalMagics']
191 184 s = '''\
192 185 a = """
193 186 ,1,2,3
194 187 """'''
195 188 ip.user_ns.pop('foo', None)
196 189 tm.store_or_execute(s, 'foo')
197 190 nt.assert_in('foo', ip.user_ns)
198 191
199 192
200 193 def test_paste_trailing_question(self):
201 194 "Test pasting sources with trailing question marks"
202 195 tm = ip.magics_manager.registry['TerminalMagics']
203 196 s = '''\
204 197 def funcfoo():
205 198 if True: #am i true?
206 199 return 'fooresult'
207 200 '''
208 201 ip.user_ns.pop('funcfoo', None)
209 202 self.paste(s)
210 203 nt.assert_equal(ip.user_ns['funcfoo'](), 'fooresult')
@@ -1,456 +1,455 b''
1 1 """Tests for the object inspection functionality.
2 2 """
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7 from __future__ import print_function
8 8
9 9 import os
10 10 import re
11 11 import sys
12 12
13 13 import nose.tools as nt
14 14
15 15 from .. import oinspect
16 16 from IPython.core.magic import (Magics, magics_class, line_magic,
17 17 cell_magic, line_cell_magic,
18 18 register_line_magic, register_cell_magic,
19 19 register_line_cell_magic)
20 20 from decorator import decorator
21 21 from IPython.testing.decorators import skipif
22 22 from IPython.testing.tools import AssertPrints
23 23 from IPython.utils.path import compress_user
24 24 from IPython.utils import py3compat
25 25 from IPython.utils.signatures import Signature, Parameter
26 26
27 27
28 28 #-----------------------------------------------------------------------------
29 29 # Globals and constants
30 30 #-----------------------------------------------------------------------------
31 31
32 32 inspector = oinspect.Inspector()
33 33 ip = get_ipython()
34 34
35 35 #-----------------------------------------------------------------------------
36 36 # Local utilities
37 37 #-----------------------------------------------------------------------------
38 38
39 39 # WARNING: since this test checks the line number where a function is
40 40 # defined, if any code is inserted above, the following line will need to be
41 41 # updated. Do NOT insert any whitespace between the next line and the function
42 42 # definition below.
43 43 THIS_LINE_NUMBER = 43 # Put here the actual number of this line
44 44
45 45 from unittest import TestCase
46 46
47 47 class Test(TestCase):
48 48
49 49 def test_find_source_lines(self):
50 50 self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines),
51 51 THIS_LINE_NUMBER+6)
52 52
53 53
54 54 # A couple of utilities to ensure these tests work the same from a source or a
55 55 # binary install
56 56 def pyfile(fname):
57 57 return os.path.normcase(re.sub('.py[co]$', '.py', fname))
58 58
59 59
60 60 def match_pyfiles(f1, f2):
61 61 nt.assert_equal(pyfile(f1), pyfile(f2))
62 62
63 63
64 64 def test_find_file():
65 65 match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__))
66 66
67 67
68 68 def test_find_file_decorated1():
69 69
70 70 @decorator
71 71 def noop1(f):
72 72 def wrapper():
73 73 return f(*a, **kw)
74 74 return wrapper
75 75
76 76 @noop1
77 77 def f(x):
78 78 "My docstring"
79 79
80 80 match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
81 81 nt.assert_equal(f.__doc__, "My docstring")
82 82
83 83
84 84 def test_find_file_decorated2():
85 85
86 86 @decorator
87 87 def noop2(f, *a, **kw):
88 88 return f(*a, **kw)
89 89
90 90 @noop2
91 91 @noop2
92 92 @noop2
93 93 def f(x):
94 94 "My docstring 2"
95 95
96 96 match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
97 97 nt.assert_equal(f.__doc__, "My docstring 2")
98 98
99 99
100 100 def test_find_file_magic():
101 101 run = ip.find_line_magic('run')
102 102 nt.assert_not_equal(oinspect.find_file(run), None)
103 103
104 104
105 105 # A few generic objects we can then inspect in the tests below
106 106
107 107 class Call(object):
108 108 """This is the class docstring."""
109 109
110 110 def __init__(self, x, y=1):
111 111 """This is the constructor docstring."""
112 112
113 113 def __call__(self, *a, **kw):
114 114 """This is the call docstring."""
115 115
116 116 def method(self, x, z=2):
117 117 """Some method's docstring"""
118 118
119 119 class HasSignature(object):
120 120 """This is the class docstring."""
121 121 __signature__ = Signature([Parameter('test', Parameter.POSITIONAL_OR_KEYWORD)])
122 122
123 123 def __init__(self, *args):
124 124 """This is the init docstring"""
125 125
126 126
127 127 class SimpleClass(object):
128 128 def method(self, x, z=2):
129 129 """Some method's docstring"""
130 130
131 131
132 132 class OldStyle:
133 133 """An old-style class for testing."""
134 134 pass
135 135
136 136
137 137 def f(x, y=2, *a, **kw):
138 138 """A simple function."""
139 139
140 140
141 141 def g(y, z=3, *a, **kw):
142 142 pass # no docstring
143 143
144 144
145 145 @register_line_magic
146 146 def lmagic(line):
147 147 "A line magic"
148 148
149 149
150 150 @register_cell_magic
151 151 def cmagic(line, cell):
152 152 "A cell magic"
153 153
154 154
155 155 @register_line_cell_magic
156 156 def lcmagic(line, cell=None):
157 157 "A line/cell magic"
158 158
159 159
160 160 @magics_class
161 161 class SimpleMagics(Magics):
162 162 @line_magic
163 163 def Clmagic(self, cline):
164 164 "A class-based line magic"
165 165
166 166 @cell_magic
167 167 def Ccmagic(self, cline, ccell):
168 168 "A class-based cell magic"
169 169
170 170 @line_cell_magic
171 171 def Clcmagic(self, cline, ccell=None):
172 172 "A class-based line/cell magic"
173 173
174 174
175 175 class Awkward(object):
176 176 def __getattr__(self, name):
177 177 raise Exception(name)
178 178
179 179 class NoBoolCall:
180 180 """
181 181 callable with `__bool__` raising should still be inspect-able.
182 182 """
183 183
184 184 def __call__(self):
185 185 """does nothing"""
186 186 pass
187 187
188 188 def __bool__(self):
189 189 """just raise NotImplemented"""
190 190 raise NotImplementedError('Must be implemented')
191 191
192 192
193 193 class SerialLiar(object):
194 194 """Attribute accesses always get another copy of the same class.
195 195
196 196 unittest.mock.call does something similar, but it's not ideal for testing
197 197 as the failure mode is to eat all your RAM. This gives up after 10k levels.
198 198 """
199 199 def __init__(self, max_fibbing_twig, lies_told=0):
200 200 if lies_told > 10000:
201 201 raise RuntimeError('Nose too long, honesty is the best policy')
202 202 self.max_fibbing_twig = max_fibbing_twig
203 203 self.lies_told = lies_told
204 204 max_fibbing_twig[0] = max(max_fibbing_twig[0], lies_told)
205 205
206 206 def __getattr__(self, item):
207 207 return SerialLiar(self.max_fibbing_twig, self.lies_told + 1)
208 208
209 209
210 210 def check_calltip(obj, name, call, docstring):
211 211 """Generic check pattern all calltip tests will use"""
212 212 info = inspector.info(obj, name)
213 213 call_line, ds = oinspect.call_tip(info)
214 214 nt.assert_equal(call_line, call)
215 215 nt.assert_equal(ds, docstring)
216 216
217 217 #-----------------------------------------------------------------------------
218 218 # Tests
219 219 #-----------------------------------------------------------------------------
220 220
221 221 def test_calltip_class():
222 222 check_calltip(Call, 'Call', 'Call(x, y=1)', Call.__init__.__doc__)
223 223
224 224
225 225 def test_calltip_instance():
226 226 c = Call(1)
227 227 check_calltip(c, 'c', 'c(*a, **kw)', c.__call__.__doc__)
228 228
229 229
230 230 def test_calltip_method():
231 231 c = Call(1)
232 232 check_calltip(c.method, 'c.method', 'c.method(x, z=2)', c.method.__doc__)
233 233
234 234
235 235 def test_calltip_function():
236 236 check_calltip(f, 'f', 'f(x, y=2, *a, **kw)', f.__doc__)
237 237
238 238
239 239 def test_calltip_function2():
240 240 check_calltip(g, 'g', 'g(y, z=3, *a, **kw)', '<no docstring>')
241 241
242 242
243 243 @skipif(sys.version_info >= (3, 5))
244 244 def test_calltip_builtin():
245 245 check_calltip(sum, 'sum', None, sum.__doc__)
246 246
247 247
248 248 def test_calltip_line_magic():
249 249 check_calltip(lmagic, 'lmagic', 'lmagic(line)', "A line magic")
250 250
251 251
252 252 def test_calltip_cell_magic():
253 253 check_calltip(cmagic, 'cmagic', 'cmagic(line, cell)', "A cell magic")
254 254
255 255
256 256 def test_calltip_line_cell_magic():
257 257 check_calltip(lcmagic, 'lcmagic', 'lcmagic(line, cell=None)',
258 258 "A line/cell magic")
259 259
260 260
261 261 def test_class_magics():
262 262 cm = SimpleMagics(ip)
263 263 ip.register_magics(cm)
264 264 check_calltip(cm.Clmagic, 'Clmagic', 'Clmagic(cline)',
265 265 "A class-based line magic")
266 266 check_calltip(cm.Ccmagic, 'Ccmagic', 'Ccmagic(cline, ccell)',
267 267 "A class-based cell magic")
268 268 check_calltip(cm.Clcmagic, 'Clcmagic', 'Clcmagic(cline, ccell=None)',
269 269 "A class-based line/cell magic")
270 270
271 271
272 272 def test_info():
273 273 "Check that Inspector.info fills out various fields as expected."
274 274 i = inspector.info(Call, oname='Call')
275 275 nt.assert_equal(i['type_name'], 'type')
276 276 expted_class = str(type(type)) # <class 'type'> (Python 3) or <type 'type'>
277 277 nt.assert_equal(i['base_class'], expted_class)
278 if sys.version_info > (3,):
279 nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
278 nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
280 279 fname = __file__
281 280 if fname.endswith(".pyc"):
282 281 fname = fname[:-1]
283 282 # case-insensitive comparison needed on some filesystems
284 283 # e.g. Windows:
285 284 nt.assert_equal(i['file'].lower(), compress_user(fname).lower())
286 285 nt.assert_equal(i['definition'], None)
287 286 nt.assert_equal(i['docstring'], Call.__doc__)
288 287 nt.assert_equal(i['source'], None)
289 288 nt.assert_true(i['isclass'])
290 289 _self_py2 = '' if py3compat.PY3 else 'self, '
291 290 nt.assert_equal(i['init_definition'], "Call(%sx, y=1)" % _self_py2)
292 291 nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
293 292
294 293 i = inspector.info(Call, detail_level=1)
295 294 nt.assert_not_equal(i['source'], None)
296 295 nt.assert_equal(i['docstring'], None)
297 296
298 297 c = Call(1)
299 298 c.__doc__ = "Modified instance docstring"
300 299 i = inspector.info(c)
301 300 nt.assert_equal(i['type_name'], 'Call')
302 301 nt.assert_equal(i['docstring'], "Modified instance docstring")
303 302 nt.assert_equal(i['class_docstring'], Call.__doc__)
304 303 nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
305 304 nt.assert_equal(i['call_docstring'], Call.__call__.__doc__)
306 305
307 306 # Test old-style classes, which for example may not have an __init__ method.
308 307 if not py3compat.PY3:
309 308 i = inspector.info(OldStyle)
310 309 nt.assert_equal(i['type_name'], 'classobj')
311 310
312 311 i = inspector.info(OldStyle())
313 312 nt.assert_equal(i['type_name'], 'instance')
314 313 nt.assert_equal(i['docstring'], OldStyle.__doc__)
315 314
316 315 def test_class_signature():
317 316 info = inspector.info(HasSignature, 'HasSignature')
318 317 nt.assert_equal(info['init_definition'], "HasSignature(test)")
319 318 nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__)
320 319
321 320 def test_info_awkward():
322 321 # Just test that this doesn't throw an error.
323 322 inspector.info(Awkward())
324 323
325 324 def test_bool_raise():
326 325 inspector.info(NoBoolCall())
327 326
328 327 def test_info_serialliar():
329 328 fib_tracker = [0]
330 329 inspector.info(SerialLiar(fib_tracker))
331 330
332 331 # Nested attribute access should be cut off at 100 levels deep to avoid
333 332 # infinite loops: https://github.com/ipython/ipython/issues/9122
334 333 nt.assert_less(fib_tracker[0], 9000)
335 334
336 335 def test_calldef_none():
337 336 # We should ignore __call__ for all of these.
338 337 for obj in [f, SimpleClass().method, any, str.upper]:
339 338 print(obj)
340 339 i = inspector.info(obj)
341 340 nt.assert_is(i['call_def'], None)
342 341
343 342 def f_kwarg(pos, *, kwonly):
344 343 pass
345 344
346 345 def test_definition_kwonlyargs():
347 346 i = inspector.info(f_kwarg, oname='f_kwarg') # analysis:ignore
348 347 nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
349 348
350 349 def test_getdoc():
351 350 class A(object):
352 351 """standard docstring"""
353 352 pass
354 353
355 354 class B(object):
356 355 """standard docstring"""
357 356 def getdoc(self):
358 357 return "custom docstring"
359 358
360 359 class C(object):
361 360 """standard docstring"""
362 361 def getdoc(self):
363 362 return None
364 363
365 364 a = A()
366 365 b = B()
367 366 c = C()
368 367
369 368 nt.assert_equal(oinspect.getdoc(a), "standard docstring")
370 369 nt.assert_equal(oinspect.getdoc(b), "custom docstring")
371 370 nt.assert_equal(oinspect.getdoc(c), "standard docstring")
372 371
373 372
374 373 def test_empty_property_has_no_source():
375 374 i = inspector.info(property(), detail_level=1)
376 375 nt.assert_is(i['source'], None)
377 376
378 377
379 378 def test_property_sources():
380 379 import zlib
381 380
382 381 class A(object):
383 382 @property
384 383 def foo(self):
385 384 return 'bar'
386 385
387 386 foo = foo.setter(lambda self, v: setattr(self, 'bar', v))
388 387
389 388 id = property(id)
390 389 compress = property(zlib.compress)
391 390
392 391 i = inspector.info(A.foo, detail_level=1)
393 392 nt.assert_in('def foo(self):', i['source'])
394 393 nt.assert_in('lambda self, v:', i['source'])
395 394
396 395 i = inspector.info(A.id, detail_level=1)
397 396 nt.assert_in('fget = <function id>', i['source'])
398 397
399 398 i = inspector.info(A.compress, detail_level=1)
400 399 nt.assert_in('fget = <function zlib.compress>', i['source'])
401 400
402 401
403 402 def test_property_docstring_is_in_info_for_detail_level_0():
404 403 class A(object):
405 404 @property
406 405 def foobar(self):
407 406 """This is `foobar` property."""
408 407 pass
409 408
410 409 ip.user_ns['a_obj'] = A()
411 410 nt.assert_equals(
412 411 'This is `foobar` property.',
413 412 ip.object_inspect('a_obj.foobar', detail_level=0)['docstring'])
414 413
415 414 ip.user_ns['a_cls'] = A
416 415 nt.assert_equals(
417 416 'This is `foobar` property.',
418 417 ip.object_inspect('a_cls.foobar', detail_level=0)['docstring'])
419 418
420 419
421 420 def test_pdef():
422 421 # See gh-1914
423 422 def foo(): pass
424 423 inspector.pdef(foo, 'foo')
425 424
426 425
427 426 def test_pinfo_nonascii():
428 427 # See gh-1177
429 428 from . import nonascii2
430 429 ip.user_ns['nonascii2'] = nonascii2
431 430 ip._inspect('pinfo', 'nonascii2', detail_level=1)
432 431
433 432
434 433 def test_pinfo_magic():
435 434 with AssertPrints('Docstring:'):
436 435 ip._inspect('pinfo', 'lsmagic', detail_level=0)
437 436
438 437 with AssertPrints('Source:'):
439 438 ip._inspect('pinfo', 'lsmagic', detail_level=1)
440 439
441 440
442 441 def test_init_colors():
443 442 # ensure colors are not present in signature info
444 443 info = inspector.info(HasSignature)
445 444 init_def = info['init_definition']
446 445 nt.assert_not_in('[0m', init_def)
447 446
448 447
449 448 def test_builtin_init():
450 449 info = inspector.info(list)
451 450 init_def = info['init_definition']
452 451 # Python < 3.4 can't get init definition from builtins,
453 452 # but still exercise the inspection in case of error-raising bugs.
454 453 if sys.version_info >= (3,4):
455 454 nt.assert_is_not_none(init_def)
456 455
@@ -1,356 +1,355 b''
1 1 # encoding: utf-8
2 2 """Tests for IPython.core.ultratb
3 3 """
4 4 import io
5 5 import sys
6 6 import os.path
7 7 from textwrap import dedent
8 8 import traceback
9 9 import unittest
10 10
11 11 try:
12 12 from unittest import mock
13 13 except ImportError:
14 14 import mock # Python 2
15 15
16 16 from ..ultratb import ColorTB, VerboseTB, find_recursion
17 17
18 18
19 19 from IPython.testing import tools as tt
20 20 from IPython.testing.decorators import onlyif_unicode_paths
21 21 from IPython.utils.syspathcontext import prepended_to_syspath
22 22 from IPython.utils.tempdir import TemporaryDirectory
23 23 from IPython.utils.py3compat import PY3
24 24
25 25 ip = get_ipython()
26 26
27 27 file_1 = """1
28 28 2
29 29 3
30 30 def f():
31 31 1/0
32 32 """
33 33
34 34 file_2 = """def f():
35 35 1/0
36 36 """
37 37
38 38 class ChangedPyFileTest(unittest.TestCase):
39 39 def test_changing_py_file(self):
40 40 """Traceback produced if the line where the error occurred is missing?
41 41
42 42 https://github.com/ipython/ipython/issues/1456
43 43 """
44 44 with TemporaryDirectory() as td:
45 45 fname = os.path.join(td, "foo.py")
46 46 with open(fname, "w") as f:
47 47 f.write(file_1)
48 48
49 49 with prepended_to_syspath(td):
50 50 ip.run_cell("import foo")
51 51
52 52 with tt.AssertPrints("ZeroDivisionError"):
53 53 ip.run_cell("foo.f()")
54 54
55 55 # Make the file shorter, so the line of the error is missing.
56 56 with open(fname, "w") as f:
57 57 f.write(file_2)
58 58
59 59 # For some reason, this was failing on the *second* call after
60 60 # changing the file, so we call f() twice.
61 61 with tt.AssertNotPrints("Internal Python error", channel='stderr'):
62 62 with tt.AssertPrints("ZeroDivisionError"):
63 63 ip.run_cell("foo.f()")
64 64 with tt.AssertPrints("ZeroDivisionError"):
65 65 ip.run_cell("foo.f()")
66 66
67 67 iso_8859_5_file = u'''# coding: iso-8859-5
68 68
69 69 def fail():
70 70 """Π΄Π±Π˜Π–"""
71 71 1/0 # Π΄Π±Π˜Π–
72 72 '''
73 73
74 74 class NonAsciiTest(unittest.TestCase):
75 75 @onlyif_unicode_paths
76 76 def test_nonascii_path(self):
77 77 # Non-ascii directory name as well.
78 78 with TemporaryDirectory(suffix=u'Γ©') as td:
79 79 fname = os.path.join(td, u"fooΓ©.py")
80 80 with open(fname, "w") as f:
81 81 f.write(file_1)
82 82
83 83 with prepended_to_syspath(td):
84 84 ip.run_cell("import foo")
85 85
86 86 with tt.AssertPrints("ZeroDivisionError"):
87 87 ip.run_cell("foo.f()")
88 88
89 89 def test_iso8859_5(self):
90 90 with TemporaryDirectory() as td:
91 91 fname = os.path.join(td, 'dfghjkl.py')
92 92
93 93 with io.open(fname, 'w', encoding='iso-8859-5') as f:
94 94 f.write(iso_8859_5_file)
95 95
96 96 with prepended_to_syspath(td):
97 97 ip.run_cell("from dfghjkl import fail")
98 98
99 99 with tt.AssertPrints("ZeroDivisionError"):
100 100 with tt.AssertPrints(u'Π΄Π±Π˜Π–', suppress=False):
101 101 ip.run_cell('fail()')
102 102
103 103 def test_nonascii_msg(self):
104 104 cell = u"raise Exception('Γ©')"
105 105 expected = u"Exception('Γ©')"
106 106 ip.run_cell("%xmode plain")
107 107 with tt.AssertPrints(expected):
108 108 ip.run_cell(cell)
109 109
110 110 ip.run_cell("%xmode verbose")
111 111 with tt.AssertPrints(expected):
112 112 ip.run_cell(cell)
113 113
114 114 ip.run_cell("%xmode context")
115 115 with tt.AssertPrints(expected):
116 116 ip.run_cell(cell)
117 117
118 118
119 119 class NestedGenExprTestCase(unittest.TestCase):
120 120 """
121 121 Regression test for the following issues:
122 122 https://github.com/ipython/ipython/issues/8293
123 123 https://github.com/ipython/ipython/issues/8205
124 124 """
125 125 def test_nested_genexpr(self):
126 126 code = dedent(
127 127 """\
128 128 class SpecificException(Exception):
129 129 pass
130 130
131 131 def foo(x):
132 132 raise SpecificException("Success!")
133 133
134 134 sum(sum(foo(x) for _ in [0]) for x in [0])
135 135 """
136 136 )
137 137 with tt.AssertPrints('SpecificException: Success!', suppress=False):
138 138 ip.run_cell(code)
139 139
140 140
141 141 indentationerror_file = """if True:
142 142 zoon()
143 143 """
144 144
145 145 class IndentationErrorTest(unittest.TestCase):
146 146 def test_indentationerror_shows_line(self):
147 147 # See issue gh-2398
148 148 with tt.AssertPrints("IndentationError"):
149 149 with tt.AssertPrints("zoon()", suppress=False):
150 150 ip.run_cell(indentationerror_file)
151 151
152 152 with TemporaryDirectory() as td:
153 153 fname = os.path.join(td, "foo.py")
154 154 with open(fname, "w") as f:
155 155 f.write(indentationerror_file)
156 156
157 157 with tt.AssertPrints("IndentationError"):
158 158 with tt.AssertPrints("zoon()", suppress=False):
159 159 ip.magic('run %s' % fname)
160 160
161 161 se_file_1 = """1
162 162 2
163 163 7/
164 164 """
165 165
166 166 se_file_2 = """7/
167 167 """
168 168
169 169 class SyntaxErrorTest(unittest.TestCase):
170 170 def test_syntaxerror_without_lineno(self):
171 171 with tt.AssertNotPrints("TypeError"):
172 172 with tt.AssertPrints("line unknown"):
173 173 ip.run_cell("raise SyntaxError()")
174 174
175 175 def test_changing_py_file(self):
176 176 with TemporaryDirectory() as td:
177 177 fname = os.path.join(td, "foo.py")
178 178 with open(fname, 'w') as f:
179 179 f.write(se_file_1)
180 180
181 181 with tt.AssertPrints(["7/", "SyntaxError"]):
182 182 ip.magic("run " + fname)
183 183
184 184 # Modify the file
185 185 with open(fname, 'w') as f:
186 186 f.write(se_file_2)
187 187
188 188 # The SyntaxError should point to the correct line
189 189 with tt.AssertPrints(["7/", "SyntaxError"]):
190 190 ip.magic("run " + fname)
191 191
192 192 def test_non_syntaxerror(self):
193 193 # SyntaxTB may be called with an error other than a SyntaxError
194 194 # See e.g. gh-4361
195 195 try:
196 196 raise ValueError('QWERTY')
197 197 except ValueError:
198 198 with tt.AssertPrints('QWERTY'):
199 199 ip.showsyntaxerror()
200 200
201 201
202 202 class Python3ChainedExceptionsTest(unittest.TestCase):
203 203 DIRECT_CAUSE_ERROR_CODE = """
204 204 try:
205 205 x = 1 + 2
206 206 print(not_defined_here)
207 207 except Exception as e:
208 208 x += 55
209 209 x - 1
210 210 y = {}
211 211 raise KeyError('uh') from e
212 212 """
213 213
214 214 EXCEPTION_DURING_HANDLING_CODE = """
215 215 try:
216 216 x = 1 + 2
217 217 print(not_defined_here)
218 218 except Exception as e:
219 219 x += 55
220 220 x - 1
221 221 y = {}
222 222 raise KeyError('uh')
223 223 """
224 224
225 225 SUPPRESS_CHAINING_CODE = """
226 226 try:
227 227 1/0
228 228 except Exception:
229 229 raise ValueError("Yikes") from None
230 230 """
231 231
232 232 def test_direct_cause_error(self):
233 233 if PY3:
234 234 with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
235 235 ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
236 236
237 237 def test_exception_during_handling_error(self):
238 238 if PY3:
239 239 with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
240 240 ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
241 241
242 242 def test_suppress_exception_chaining(self):
243 243 if PY3:
244 244 with tt.AssertNotPrints("ZeroDivisionError"), \
245 245 tt.AssertPrints("ValueError", suppress=False):
246 246 ip.run_cell(self.SUPPRESS_CHAINING_CODE)
247 247
248 248
249 249 class RecursionTest(unittest.TestCase):
250 250 DEFINITIONS = """
251 251 def non_recurs():
252 252 1/0
253 253
254 254 def r1():
255 255 r1()
256 256
257 257 def r3a():
258 258 r3b()
259 259
260 260 def r3b():
261 261 r3c()
262 262
263 263 def r3c():
264 264 r3a()
265 265
266 266 def r3o1():
267 267 r3a()
268 268
269 269 def r3o2():
270 270 r3o1()
271 271 """
272 272 def setUp(self):
273 273 ip.run_cell(self.DEFINITIONS)
274 274
275 275 def test_no_recursion(self):
276 276 with tt.AssertNotPrints("frames repeated"):
277 277 ip.run_cell("non_recurs()")
278 278
279 279 def test_recursion_one_frame(self):
280 280 with tt.AssertPrints("1 frames repeated"):
281 281 ip.run_cell("r1()")
282 282
283 283 def test_recursion_three_frames(self):
284 284 with tt.AssertPrints("3 frames repeated"):
285 285 ip.run_cell("r3o2()")
286 286
287 287 def test_find_recursion(self):
288 288 captured = []
289 289 def capture_exc(*args, **kwargs):
290 290 captured.append(sys.exc_info())
291 291 with mock.patch.object(ip, 'showtraceback', capture_exc):
292 292 ip.run_cell("r3o2()")
293 293
294 294 self.assertEqual(len(captured), 1)
295 295 etype, evalue, tb = captured[0]
296 296 self.assertIn("recursion", str(evalue))
297 297
298 298 records = ip.InteractiveTB.get_records(tb, 3, ip.InteractiveTB.tb_offset)
299 299 for r in records[:10]:
300 300 print(r[1:4])
301 301
302 302 # The outermost frames should be:
303 303 # 0: the 'cell' that was running when the exception came up
304 304 # 1: r3o2()
305 305 # 2: r3o1()
306 306 # 3: r3a()
307 307 # Then repeating r3b, r3c, r3a
308 308 last_unique, repeat_length = find_recursion(etype, evalue, records)
309 309 self.assertEqual(last_unique, 2)
310 310 self.assertEqual(repeat_length, 3)
311 311
312 312
313 313 #----------------------------------------------------------------------------
314 314
315 315 # module testing (minimal)
316 if sys.version_info > (3,):
317 def test_handlers():
318 def spam(c, d_e):
319 (d, e) = d_e
320 x = c + d
321 y = c * d
322 foo(x, y)
323
324 def foo(a, b, bar=1):
325 eggs(a, b + bar)
326
327 def eggs(f, g, z=globals()):
328 h = f + g
329 i = f - g
330 return h / i
331
332 buff = io.StringIO()
333
334 buff.write('')
335 buff.write('*** Before ***')
336 try:
337 buff.write(spam(1, (2, 3)))
338 except:
339 traceback.print_exc(file=buff)
340
341 handler = ColorTB(ostream=buff)
342 buff.write('*** ColorTB ***')
343 try:
344 buff.write(spam(1, (2, 3)))
345 except:
346 handler(*sys.exc_info())
347 buff.write('')
348
349 handler = VerboseTB(ostream=buff)
350 buff.write('*** VerboseTB ***')
351 try:
352 buff.write(spam(1, (2, 3)))
353 except:
354 handler(*sys.exc_info())
355 buff.write('')
316 def test_handlers():
317 def spam(c, d_e):
318 (d, e) = d_e
319 x = c + d
320 y = c * d
321 foo(x, y)
322
323 def foo(a, b, bar=1):
324 eggs(a, b + bar)
325
326 def eggs(f, g, z=globals()):
327 h = f + g
328 i = f - g
329 return h / i
330
331 buff = io.StringIO()
332
333 buff.write('')
334 buff.write('*** Before ***')
335 try:
336 buff.write(spam(1, (2, 3)))
337 except:
338 traceback.print_exc(file=buff)
339
340 handler = ColorTB(ostream=buff)
341 buff.write('*** ColorTB ***')
342 try:
343 buff.write(spam(1, (2, 3)))
344 except:
345 handler(*sys.exc_info())
346 buff.write('')
347
348 handler = VerboseTB(ostream=buff)
349 buff.write('*** VerboseTB ***')
350 try:
351 buff.write(spam(1, (2, 3)))
352 except:
353 handler(*sys.exc_info())
354 buff.write('')
356 355
@@ -1,433 +1,432 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 # Copyright (c) IPython Development Team.
18 18 # Distributed under the terms of the Modified BSD License.
19 19
20 20 from __future__ import print_function
21 21
22 22 import glob
23 23 from io import BytesIO
24 24 import os
25 25 import os.path as path
26 26 import sys
27 27 from threading import Thread, Lock, Event
28 28 import warnings
29 29
30 30 import nose.plugins.builtin
31 31 from nose.plugins.xunit import Xunit
32 32 from nose import SkipTest
33 33 from nose.core import TestProgram
34 34 from nose.plugins import Plugin
35 35 from nose.util import safe_str
36 36
37 37 from IPython import version_info
38 38 from IPython.utils.py3compat import bytes_to_str
39 39 from IPython.utils.importstring import import_item
40 40 from IPython.testing.plugin.ipdoctest import IPythonDoctest
41 41 from IPython.external.decorators import KnownFailure, knownfailureif
42 42
43 43 pjoin = path.join
44 44
45 45
46 46 # Enable printing all warnings raise by IPython's modules
47 47 warnings.filterwarnings('ignore', message='.*Matplotlib is building the font cache.*', category=UserWarning, module='.*')
48 if sys.version_info > (3,0):
49 warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
48 warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
50 49 warnings.filterwarnings('error', message=".*{'config': True}.*", category=DeprecationWarning, module='IPy.*')
51 50 warnings.filterwarnings('default', message='.*', category=Warning, module='IPy.*')
52 51
53 52 if version_info < (6,):
54 53 # nose.tools renames all things from `camelCase` to `snake_case` which raise an
55 54 # warning with the runner they also import from standard import library. (as of Dec 2015)
56 55 # Ignore, let's revisit that in a couple of years for IPython 6.
57 56 warnings.filterwarnings('ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')
58 57
59 58
60 59 # ------------------------------------------------------------------------------
61 60 # Monkeypatch Xunit to count known failures as skipped.
62 61 # ------------------------------------------------------------------------------
63 62 def monkeypatch_xunit():
64 63 try:
65 64 knownfailureif(True)(lambda: None)()
66 65 except Exception as e:
67 66 KnownFailureTest = type(e)
68 67
69 68 def addError(self, test, err, capt=None):
70 69 if issubclass(err[0], KnownFailureTest):
71 70 err = (SkipTest,) + err[1:]
72 71 return self.orig_addError(test, err, capt)
73 72
74 73 Xunit.orig_addError = Xunit.addError
75 74 Xunit.addError = addError
76 75
77 76 #-----------------------------------------------------------------------------
78 77 # Check which dependencies are installed and greater than minimum version.
79 78 #-----------------------------------------------------------------------------
80 79 def extract_version(mod):
81 80 return mod.__version__
82 81
83 82 def test_for(item, min_version=None, callback=extract_version):
84 83 """Test to see if item is importable, and optionally check against a minimum
85 84 version.
86 85
87 86 If min_version is given, the default behavior is to check against the
88 87 `__version__` attribute of the item, but specifying `callback` allows you to
89 88 extract the value you are interested in. e.g::
90 89
91 90 In [1]: import sys
92 91
93 92 In [2]: from IPython.testing.iptest import test_for
94 93
95 94 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
96 95 Out[3]: True
97 96
98 97 """
99 98 try:
100 99 check = import_item(item)
101 100 except (ImportError, RuntimeError):
102 101 # GTK reports Runtime error if it can't be initialized even if it's
103 102 # importable.
104 103 return False
105 104 else:
106 105 if min_version:
107 106 if callback:
108 107 # extra processing step to get version to compare
109 108 check = callback(check)
110 109
111 110 return check >= min_version
112 111 else:
113 112 return True
114 113
115 114 # Global dict where we can store information on what we have and what we don't
116 115 # have available at test run time
117 116 have = {'matplotlib': test_for('matplotlib'),
118 117 'pygments': test_for('pygments'),
119 118 'sqlite3': test_for('sqlite3')}
120 119
121 120 #-----------------------------------------------------------------------------
122 121 # Test suite definitions
123 122 #-----------------------------------------------------------------------------
124 123
125 124 test_group_names = ['core',
126 125 'extensions', 'lib', 'terminal', 'testing', 'utils',
127 126 ]
128 127
129 128 class TestSection(object):
130 129 def __init__(self, name, includes):
131 130 self.name = name
132 131 self.includes = includes
133 132 self.excludes = []
134 133 self.dependencies = []
135 134 self.enabled = True
136 135
137 136 def exclude(self, module):
138 137 if not module.startswith('IPython'):
139 138 module = self.includes[0] + "." + module
140 139 self.excludes.append(module.replace('.', os.sep))
141 140
142 141 def requires(self, *packages):
143 142 self.dependencies.extend(packages)
144 143
145 144 @property
146 145 def will_run(self):
147 146 return self.enabled and all(have[p] for p in self.dependencies)
148 147
149 148 # Name -> (include, exclude, dependencies_met)
150 149 test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names}
151 150
152 151
153 152 # Exclusions and dependencies
154 153 # ---------------------------
155 154
156 155 # core:
157 156 sec = test_sections['core']
158 157 if not have['sqlite3']:
159 158 sec.exclude('tests.test_history')
160 159 sec.exclude('history')
161 160 if not have['matplotlib']:
162 161 sec.exclude('pylabtools'),
163 162 sec.exclude('tests.test_pylabtools')
164 163
165 164 # lib:
166 165 sec = test_sections['lib']
167 166 sec.exclude('kernel')
168 167 if not have['pygments']:
169 168 sec.exclude('tests.test_lexers')
170 169 # We do this unconditionally, so that the test suite doesn't import
171 170 # gtk, changing the default encoding and masking some unicode bugs.
172 171 sec.exclude('inputhookgtk')
173 172 # We also do this unconditionally, because wx can interfere with Unix signals.
174 173 # There are currently no tests for it anyway.
175 174 sec.exclude('inputhookwx')
176 175 # Testing inputhook will need a lot of thought, to figure out
177 176 # how to have tests that don't lock up with the gui event
178 177 # loops in the picture
179 178 sec.exclude('inputhook')
180 179
181 180 # testing:
182 181 sec = test_sections['testing']
183 182 # These have to be skipped on win32 because they use echo, rm, cd, etc.
184 183 # See ticket https://github.com/ipython/ipython/issues/87
185 184 if sys.platform == 'win32':
186 185 sec.exclude('plugin.test_exampleip')
187 186 sec.exclude('plugin.dtexample')
188 187
189 188 # don't run jupyter_console tests found via shim
190 189 test_sections['terminal'].exclude('console')
191 190
192 191 # extensions:
193 192 sec = test_sections['extensions']
194 193 # This is deprecated in favour of rpy2
195 194 sec.exclude('rmagic')
196 195 # autoreload does some strange stuff, so move it to its own test section
197 196 sec.exclude('autoreload')
198 197 sec.exclude('tests.test_autoreload')
199 198 test_sections['autoreload'] = TestSection('autoreload',
200 199 ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
201 200 test_group_names.append('autoreload')
202 201
203 202
204 203 #-----------------------------------------------------------------------------
205 204 # Functions and classes
206 205 #-----------------------------------------------------------------------------
207 206
208 207 def check_exclusions_exist():
209 208 from IPython.paths import get_ipython_package_dir
210 209 from warnings import warn
211 210 parent = os.path.dirname(get_ipython_package_dir())
212 211 for sec in test_sections:
213 212 for pattern in sec.exclusions:
214 213 fullpath = pjoin(parent, pattern)
215 214 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
216 215 warn("Excluding nonexistent file: %r" % pattern)
217 216
218 217
219 218 class ExclusionPlugin(Plugin):
220 219 """A nose plugin to effect our exclusions of files and directories.
221 220 """
222 221 name = 'exclusions'
223 222 score = 3000 # Should come before any other plugins
224 223
225 224 def __init__(self, exclude_patterns=None):
226 225 """
227 226 Parameters
228 227 ----------
229 228
230 229 exclude_patterns : sequence of strings, optional
231 230 Filenames containing these patterns (as raw strings, not as regular
232 231 expressions) are excluded from the tests.
233 232 """
234 233 self.exclude_patterns = exclude_patterns or []
235 234 super(ExclusionPlugin, self).__init__()
236 235
237 236 def options(self, parser, env=os.environ):
238 237 Plugin.options(self, parser, env)
239 238
240 239 def configure(self, options, config):
241 240 Plugin.configure(self, options, config)
242 241 # Override nose trying to disable plugin.
243 242 self.enabled = True
244 243
245 244 def wantFile(self, filename):
246 245 """Return whether the given filename should be scanned for tests.
247 246 """
248 247 if any(pat in filename for pat in self.exclude_patterns):
249 248 return False
250 249 return None
251 250
252 251 def wantDirectory(self, directory):
253 252 """Return whether the given directory should be scanned for tests.
254 253 """
255 254 if any(pat in directory for pat in self.exclude_patterns):
256 255 return False
257 256 return None
258 257
259 258
260 259 class StreamCapturer(Thread):
261 260 daemon = True # Don't hang if main thread crashes
262 261 started = False
263 262 def __init__(self, echo=False):
264 263 super(StreamCapturer, self).__init__()
265 264 self.echo = echo
266 265 self.streams = []
267 266 self.buffer = BytesIO()
268 267 self.readfd, self.writefd = os.pipe()
269 268 self.buffer_lock = Lock()
270 269 self.stop = Event()
271 270
272 271 def run(self):
273 272 self.started = True
274 273
275 274 while not self.stop.is_set():
276 275 chunk = os.read(self.readfd, 1024)
277 276
278 277 with self.buffer_lock:
279 278 self.buffer.write(chunk)
280 279 if self.echo:
281 280 sys.stdout.write(bytes_to_str(chunk))
282 281
283 282 os.close(self.readfd)
284 283 os.close(self.writefd)
285 284
286 285 def reset_buffer(self):
287 286 with self.buffer_lock:
288 287 self.buffer.truncate(0)
289 288 self.buffer.seek(0)
290 289
291 290 def get_buffer(self):
292 291 with self.buffer_lock:
293 292 return self.buffer.getvalue()
294 293
295 294 def ensure_started(self):
296 295 if not self.started:
297 296 self.start()
298 297
299 298 def halt(self):
300 299 """Safely stop the thread."""
301 300 if not self.started:
302 301 return
303 302
304 303 self.stop.set()
305 304 os.write(self.writefd, b'\0') # Ensure we're not locked in a read()
306 305 self.join()
307 306
308 307 class SubprocessStreamCapturePlugin(Plugin):
309 308 name='subprocstreams'
310 309 def __init__(self):
311 310 Plugin.__init__(self)
312 311 self.stream_capturer = StreamCapturer()
313 312 self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
314 313 # This is ugly, but distant parts of the test machinery need to be able
315 314 # to redirect streams, so we make the object globally accessible.
316 315 nose.iptest_stdstreams_fileno = self.get_write_fileno
317 316
318 317 def get_write_fileno(self):
319 318 if self.destination == 'capture':
320 319 self.stream_capturer.ensure_started()
321 320 return self.stream_capturer.writefd
322 321 elif self.destination == 'discard':
323 322 return os.open(os.devnull, os.O_WRONLY)
324 323 else:
325 324 return sys.__stdout__.fileno()
326 325
327 326 def configure(self, options, config):
328 327 Plugin.configure(self, options, config)
329 328 # Override nose trying to disable plugin.
330 329 if self.destination == 'capture':
331 330 self.enabled = True
332 331
333 332 def startTest(self, test):
334 333 # Reset log capture
335 334 self.stream_capturer.reset_buffer()
336 335
337 336 def formatFailure(self, test, err):
338 337 # Show output
339 338 ec, ev, tb = err
340 339 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
341 340 if captured.strip():
342 341 ev = safe_str(ev)
343 342 out = [ev, '>> begin captured subprocess output <<',
344 343 captured,
345 344 '>> end captured subprocess output <<']
346 345 return ec, '\n'.join(out), tb
347 346
348 347 return err
349 348
350 349 formatError = formatFailure
351 350
352 351 def finalize(self, result):
353 352 self.stream_capturer.halt()
354 353
355 354
356 355 def run_iptest():
357 356 """Run the IPython test suite using nose.
358 357
359 358 This function is called when this script is **not** called with the form
360 359 `iptest all`. It simply calls nose with appropriate command line flags
361 360 and accepts all of the standard nose arguments.
362 361 """
363 362 # Apply our monkeypatch to Xunit
364 363 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
365 364 monkeypatch_xunit()
366 365
367 366 arg1 = sys.argv[1]
368 367 if arg1 in test_sections:
369 368 section = test_sections[arg1]
370 369 sys.argv[1:2] = section.includes
371 370 elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
372 371 section = test_sections[arg1[8:]]
373 372 sys.argv[1:2] = section.includes
374 373 else:
375 374 section = TestSection(arg1, includes=[arg1])
376 375
377 376
378 377 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
379 378 # We add --exe because of setuptools' imbecility (it
380 379 # blindly does chmod +x on ALL files). Nose does the
381 380 # right thing and it tries to avoid executables,
382 381 # setuptools unfortunately forces our hand here. This
383 382 # has been discussed on the distutils list and the
384 383 # setuptools devs refuse to fix this problem!
385 384 '--exe',
386 385 ]
387 386 if '-a' not in argv and '-A' not in argv:
388 387 argv = argv + ['-a', '!crash']
389 388
390 389 if nose.__version__ >= '0.11':
391 390 # I don't fully understand why we need this one, but depending on what
392 391 # directory the test suite is run from, if we don't give it, 0 tests
393 392 # get run. Specifically, if the test suite is run from the source dir
394 393 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
395 394 # even if the same call done in this directory works fine). It appears
396 395 # that if the requested package is in the current dir, nose bails early
397 396 # by default. Since it's otherwise harmless, leave it in by default
398 397 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
399 398 argv.append('--traverse-namespace')
400 399
401 400 plugins = [ ExclusionPlugin(section.excludes), KnownFailure(),
402 401 SubprocessStreamCapturePlugin() ]
403 402
404 403 # we still have some vestigial doctests in core
405 404 if (section.name.startswith(('core', 'IPython.core'))):
406 405 plugins.append(IPythonDoctest())
407 406 argv.extend([
408 407 '--with-ipdoctest',
409 408 '--ipdoctest-tests',
410 409 '--ipdoctest-extension=txt',
411 410 ])
412 411
413 412
414 413 # Use working directory set by parent process (see iptestcontroller)
415 414 if 'IPTEST_WORKING_DIR' in os.environ:
416 415 os.chdir(os.environ['IPTEST_WORKING_DIR'])
417 416
418 417 # We need a global ipython running in this process, but the special
419 418 # in-process group spawns its own IPython kernels, so for *that* group we
420 419 # must avoid also opening the global one (otherwise there's a conflict of
421 420 # singletons). Ultimately the solution to this problem is to refactor our
422 421 # assumptions about what needs to be a singleton and what doesn't (app
423 422 # objects should, individual shells shouldn't). But for now, this
424 423 # workaround allows the test suite for the inprocess module to complete.
425 424 if 'kernel.inprocess' not in section.name:
426 425 from IPython.testing import globalipapp
427 426 globalipapp.start_ipython()
428 427
429 428 # Now nose can run
430 429 TestProgram(argv=argv, addplugins=plugins)
431 430
432 431 if __name__ == '__main__':
433 432 run_iptest()
@@ -1,43 +1,30 b''
1 """Decorators marks that a doctest should be skipped, for both python 2 and 3.
1 """Decorators marks that a doctest should be skipped.
2 2
3 3 The IPython.testing.decorators module triggers various extra imports, including
4 4 numpy and sympy if they're present. Since this decorator is used in core parts
5 5 of IPython, it's in a separate module so that running IPython doesn't trigger
6 6 those imports."""
7 7
8 #-----------------------------------------------------------------------------
9 # Copyright (C) 2009-2011 The IPython Development Team
10 #
11 # Distributed under the terms of the BSD License. The full license is in
12 # the file COPYING, distributed as part of this software.
13 #-----------------------------------------------------------------------------
8 # Copyright (C) IPython Development Team
9 # Distributed under the terms of the Modified BSD License.
14 10
15 #-----------------------------------------------------------------------------
16 # Imports
17 #-----------------------------------------------------------------------------
18
19 import sys
20
21 #-----------------------------------------------------------------------------
22 # Decorators
23 #-----------------------------------------------------------------------------
24 11
25 12 def skip_doctest(f):
26 13 """Decorator - mark a function or method for skipping its doctest.
27 14
28 15 This decorator allows you to mark a function whose docstring you wish to
29 16 omit from testing, while preserving the docstring for introspection, help,
30 17 etc."""
31 18 f.skip_doctest = True
32 19 return f
33 20
34 21
35 22 def skip_doctest_py3(f):
36 23 """Decorator - skip the doctest under Python 3."""
37 24 f.skip_doctest = (sys.version_info[0] >= 3)
38 25 return f
39 26
40 27 def skip_doctest_py2(f):
41 28 """Decorator - skip the doctest under Python 3."""
42 29 f.skip_doctest = (sys.version_info[0] < 3)
43 30 return f
This diff has been collapsed as it changes many lines, (594 lines changed) Show them Hide them
@@ -1,9 +1,595 b''
1 """Load our patched versions of tokenize.
1 """Patched version of standard library tokenize, to deal with various bugs.
2
3 Based on Python 3.2 code.
4
5 Patches:
6
7 - Gareth Rees' patch for Python issue #12691 (untokenizing)
8 - Except we don't encode the output of untokenize
9 - Python 2 compatible syntax, so that it can be byte-compiled at installation
10 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
11 on whether they are in a multi-line statement. Filed as Python issue #17061.
12 - Export generate_tokens & TokenError
13 - u and rb literals are allowed under Python 3.3 and above.
14
15 ------------------------------------------------------------------------------
16 Tokenization help for Python programs.
17
18 tokenize(readline) is a generator that breaks a stream of bytes into
19 Python tokens. It decodes the bytes according to PEP-0263 for
20 determining source file encoding.
21
22 It accepts a readline-like method which is called repeatedly to get the
23 next line of input (or b"" for EOF). It generates 5-tuples with these
24 members:
25
26 the token type (see token.py)
27 the token (a string)
28 the starting (row, column) indices of the token (a 2-tuple of ints)
29 the ending (row, column) indices of the token (a 2-tuple of ints)
30 the original line (string)
31
32 It is designed to match the working of the Python tokenizer exactly, except
33 that it produces COMMENT tokens for comments and gives type OP for all
34 operators. Additionally, all token lists start with an ENCODING token
35 which tells you which encoding was used to decode the bytes stream.
2 36 """
37 from __future__ import absolute_import
3 38
39 __author__ = 'Ka-Ping Yee <ping@lfw.org>'
40 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
41 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
42 'Michael Foord')
43 import builtins
44 import re
4 45 import sys
46 from token import *
47 from codecs import lookup, BOM_UTF8
48 import collections
49 from io import TextIOWrapper
50 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
51
52 import token
53 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
54 "NL", "untokenize", "ENCODING", "TokenInfo"]
55 del token
56
57 __all__ += ["generate_tokens", "TokenError"]
5 58
6 if sys.version_info[0] >= 3:
7 from ._tokenize_py3 import *
59 COMMENT = N_TOKENS
60 tok_name[COMMENT] = 'COMMENT'
61 NL = N_TOKENS + 1
62 tok_name[NL] = 'NL'
63 ENCODING = N_TOKENS + 2
64 tok_name[ENCODING] = 'ENCODING'
65 N_TOKENS += 3
66
67 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
68 def __repr__(self):
69 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
70 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
71 self._replace(type=annotated_type))
72
73 def group(*choices): return '(' + '|'.join(choices) + ')'
74 def any(*choices): return group(*choices) + '*'
75 def maybe(*choices): return group(*choices) + '?'
76
77 # Note: we use unicode matching for names ("\w") but ascii matching for
78 # number literals.
79 Whitespace = r'[ \f\t]*'
80 Comment = r'#[^\r\n]*'
81 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
82 Name = r'\w+'
83
84 Hexnumber = r'0[xX][0-9a-fA-F]+'
85 Binnumber = r'0[bB][01]+'
86 Octnumber = r'0[oO][0-7]+'
87 Decnumber = r'(?:0+|[1-9][0-9]*)'
88 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
89 Exponent = r'[eE][-+]?[0-9]+'
90 Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
91 Expfloat = r'[0-9]+' + Exponent
92 Floatnumber = group(Pointfloat, Expfloat)
93 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
94 Number = group(Imagnumber, Floatnumber, Intnumber)
95
96 if sys.version_info.minor >= 3:
97 StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
8 98 else:
9 from ._tokenize_py2 import *
99 StringPrefix = r'(?:[bB]?[rR]?)?'
100
101 # Tail end of ' string.
102 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
103 # Tail end of " string.
104 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
105 # Tail end of ''' string.
106 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
107 # Tail end of """ string.
108 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
109 Triple = group(StringPrefix + "'''", StringPrefix + '"""')
110 # Single-line ' or " string.
111 String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
112 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
113
114 # Because of leftmost-then-longest match semantics, be sure to put the
115 # longest operators first (e.g., if = came before ==, == would get
116 # recognized as two instances of =).
117 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
118 r"//=?", r"->",
119 r"[+\-*/%&|^=<>]=?",
120 r"~")
121
122 Bracket = '[][(){}]'
123 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
124 Funny = group(Operator, Bracket, Special)
125
126 PlainToken = group(Number, Funny, String, Name)
127 Token = Ignore + PlainToken
128
129 # First (or only) line of ' or " string.
130 ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
131 group("'", r'\\\r?\n'),
132 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
133 group('"', r'\\\r?\n'))
134 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
135 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
136
137 def _compile(expr):
138 return re.compile(expr, re.UNICODE)
139
140 tokenprog, pseudoprog, single3prog, double3prog = map(
141 _compile, (Token, PseudoToken, Single3, Double3))
142 endprogs = {"'": _compile(Single), '"': _compile(Double),
143 "'''": single3prog, '"""': double3prog,
144 "r'''": single3prog, 'r"""': double3prog,
145 "b'''": single3prog, 'b"""': double3prog,
146 "R'''": single3prog, 'R"""': double3prog,
147 "B'''": single3prog, 'B"""': double3prog,
148 "br'''": single3prog, 'br"""': double3prog,
149 "bR'''": single3prog, 'bR"""': double3prog,
150 "Br'''": single3prog, 'Br"""': double3prog,
151 "BR'''": single3prog, 'BR"""': double3prog,
152 'r': None, 'R': None, 'b': None, 'B': None}
153
154 triple_quoted = {}
155 for t in ("'''", '"""',
156 "r'''", 'r"""', "R'''", 'R"""',
157 "b'''", 'b"""', "B'''", 'B"""',
158 "br'''", 'br"""', "Br'''", 'Br"""',
159 "bR'''", 'bR"""', "BR'''", 'BR"""'):
160 triple_quoted[t] = t
161 single_quoted = {}
162 for t in ("'", '"',
163 "r'", 'r"', "R'", 'R"',
164 "b'", 'b"', "B'", 'B"',
165 "br'", 'br"', "Br'", 'Br"',
166 "bR'", 'bR"', "BR'", 'BR"' ):
167 single_quoted[t] = t
168
169 if sys.version_info.minor >= 3:
170 # Python 3.3
171 for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
172 _t2 = _prefix+'"""'
173 endprogs[_t2] = double3prog
174 triple_quoted[_t2] = _t2
175 _t1 = _prefix + "'''"
176 endprogs[_t1] = single3prog
177 triple_quoted[_t1] = _t1
178 single_quoted[_prefix+'"'] = _prefix+'"'
179 single_quoted[_prefix+"'"] = _prefix+"'"
180 del _prefix, _t2, _t1
181 endprogs['u'] = None
182 endprogs['U'] = None
183
184 del _compile
185
186 tabsize = 8
187
188 class TokenError(Exception): pass
189
190 class StopTokenizing(Exception): pass
191
192
193 class Untokenizer:
194
195 def __init__(self):
196 self.tokens = []
197 self.prev_row = 1
198 self.prev_col = 0
199 self.encoding = 'utf-8'
200
201 def add_whitespace(self, tok_type, start):
202 row, col = start
203 assert row >= self.prev_row
204 col_offset = col - self.prev_col
205 if col_offset > 0:
206 self.tokens.append(" " * col_offset)
207 elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
208 # Line was backslash-continued.
209 self.tokens.append(" ")
210
211 def untokenize(self, tokens):
212 iterable = iter(tokens)
213 for t in iterable:
214 if len(t) == 2:
215 self.compat(t, iterable)
216 break
217 tok_type, token, start, end = t[:4]
218 if tok_type == ENCODING:
219 self.encoding = token
220 continue
221 self.add_whitespace(tok_type, start)
222 self.tokens.append(token)
223 self.prev_row, self.prev_col = end
224 if tok_type in (NEWLINE, NL):
225 self.prev_row += 1
226 self.prev_col = 0
227 return "".join(self.tokens)
228
229 def compat(self, token, iterable):
230 # This import is here to avoid problems when the itertools
231 # module is not built yet and tokenize is imported.
232 from itertools import chain
233 startline = False
234 prevstring = False
235 indents = []
236 toks_append = self.tokens.append
237
238 for tok in chain([token], iterable):
239 toknum, tokval = tok[:2]
240 if toknum == ENCODING:
241 self.encoding = tokval
242 continue
243
244 if toknum in (NAME, NUMBER):
245 tokval += ' '
246
247 # Insert a space between two consecutive strings
248 if toknum == STRING:
249 if prevstring:
250 tokval = ' ' + tokval
251 prevstring = True
252 else:
253 prevstring = False
254
255 if toknum == INDENT:
256 indents.append(tokval)
257 continue
258 elif toknum == DEDENT:
259 indents.pop()
260 continue
261 elif toknum in (NEWLINE, NL):
262 startline = True
263 elif startline and indents:
264 toks_append(indents[-1])
265 startline = False
266 toks_append(tokval)
267
268
269 def untokenize(tokens):
270 """
271 Convert ``tokens`` (an iterable) back into Python source code. Return
272 a bytes object, encoded using the encoding specified by the last
273 ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
274
275 The result is guaranteed to tokenize back to match the input so that
276 the conversion is lossless and round-trips are assured. The
277 guarantee applies only to the token type and token string as the
278 spacing between tokens (column positions) may change.
279
280 :func:`untokenize` has two modes. If the input tokens are sequences
281 of length 2 (``type``, ``string``) then spaces are added as necessary to
282 preserve the round-trip property.
283
284 If the input tokens are sequences of length 4 or more (``type``,
285 ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
286 spaces are added so that each token appears in the result at the
287 position indicated by ``start`` and ``end``, if possible.
288 """
289 return Untokenizer().untokenize(tokens)
290
291
292 def _get_normal_name(orig_enc):
293 """Imitates get_normal_name in tokenizer.c."""
294 # Only care about the first 12 characters.
295 enc = orig_enc[:12].lower().replace("_", "-")
296 if enc == "utf-8" or enc.startswith("utf-8-"):
297 return "utf-8"
298 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
299 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
300 return "iso-8859-1"
301 return orig_enc
302
303 def detect_encoding(readline):
304 """
305 The detect_encoding() function is used to detect the encoding that should
306 be used to decode a Python source file. It requires one argment, readline,
307 in the same way as the tokenize() generator.
308
309 It will call readline a maximum of twice, and return the encoding used
310 (as a string) and a list of any lines (left as bytes) it has read in.
311
312 It detects the encoding from the presence of a utf-8 bom or an encoding
313 cookie as specified in pep-0263. If both a bom and a cookie are present,
314 but disagree, a SyntaxError will be raised. If the encoding cookie is an
315 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
316 'utf-8-sig' is returned.
317
318 If no encoding is specified, then the default of 'utf-8' will be returned.
319 """
320 bom_found = False
321 encoding = None
322 default = 'utf-8'
323 def read_or_stop():
324 try:
325 return readline()
326 except StopIteration:
327 return b''
328
329 def find_cookie(line):
330 try:
331 # Decode as UTF-8. Either the line is an encoding declaration,
332 # in which case it should be pure ASCII, or it must be UTF-8
333 # per default encoding.
334 line_string = line.decode('utf-8')
335 except UnicodeDecodeError:
336 raise SyntaxError("invalid or missing encoding declaration")
337
338 matches = cookie_re.findall(line_string)
339 if not matches:
340 return None
341 encoding = _get_normal_name(matches[0])
342 try:
343 codec = lookup(encoding)
344 except LookupError:
345 # This behaviour mimics the Python interpreter
346 raise SyntaxError("unknown encoding: " + encoding)
347
348 if bom_found:
349 if encoding != 'utf-8':
350 # This behaviour mimics the Python interpreter
351 raise SyntaxError('encoding problem: utf-8')
352 encoding += '-sig'
353 return encoding
354
355 first = read_or_stop()
356 if first.startswith(BOM_UTF8):
357 bom_found = True
358 first = first[3:]
359 default = 'utf-8-sig'
360 if not first:
361 return default, []
362
363 encoding = find_cookie(first)
364 if encoding:
365 return encoding, [first]
366
367 second = read_or_stop()
368 if not second:
369 return default, [first]
370
371 encoding = find_cookie(second)
372 if encoding:
373 return encoding, [first, second]
374
375 return default, [first, second]
376
377
378 def open(filename):
379 """Open a file in read only mode using the encoding detected by
380 detect_encoding().
381 """
382 buffer = builtins.open(filename, 'rb')
383 encoding, lines = detect_encoding(buffer.readline)
384 buffer.seek(0)
385 text = TextIOWrapper(buffer, encoding, line_buffering=True)
386 text.mode = 'r'
387 return text
388
389
390 def tokenize(readline):
391 """
392 The tokenize() generator requires one argment, readline, which
393 must be a callable object which provides the same interface as the
394 readline() method of built-in file objects. Each call to the function
395 should return one line of input as bytes. Alternately, readline
396 can be a callable function terminating with StopIteration:
397 readline = open(myfile, 'rb').__next__ # Example of alternate readline
398
399 The generator produces 5-tuples with these members: the token type; the
400 token string; a 2-tuple (srow, scol) of ints specifying the row and
401 column where the token begins in the source; a 2-tuple (erow, ecol) of
402 ints specifying the row and column where the token ends in the source;
403 and the line on which the token was found. The line passed is the
404 logical line; continuation lines are included.
405
406 The first token sequence will always be an ENCODING token
407 which tells you which encoding was used to decode the bytes stream.
408 """
409 # This import is here to avoid problems when the itertools module is not
410 # built yet and tokenize is imported.
411 from itertools import chain, repeat
412 encoding, consumed = detect_encoding(readline)
413 rl_gen = iter(readline, b"")
414 empty = repeat(b"")
415 return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
416
417
418 def _tokenize(readline, encoding):
419 lnum = parenlev = continued = 0
420 numchars = '0123456789'
421 contstr, needcont = '', 0
422 contline = None
423 indents = [0]
424
425 if encoding is not None:
426 if encoding == "utf-8-sig":
427 # BOM will already have been stripped.
428 encoding = "utf-8"
429 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
430 while True: # loop over lines in stream
431 try:
432 line = readline()
433 except StopIteration:
434 line = b''
435
436 if encoding is not None:
437 line = line.decode(encoding)
438 lnum += 1
439 pos, max = 0, len(line)
440
441 if contstr: # continued string
442 if not line:
443 raise TokenError("EOF in multi-line string", strstart)
444 endmatch = endprog.match(line)
445 if endmatch:
446 pos = end = endmatch.end(0)
447 yield TokenInfo(STRING, contstr + line[:end],
448 strstart, (lnum, end), contline + line)
449 contstr, needcont = '', 0
450 contline = None
451 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
452 yield TokenInfo(ERRORTOKEN, contstr + line,
453 strstart, (lnum, len(line)), contline)
454 contstr = ''
455 contline = None
456 continue
457 else:
458 contstr = contstr + line
459 contline = contline + line
460 continue
461
462 elif parenlev == 0 and not continued: # new statement
463 if not line: break
464 column = 0
465 while pos < max: # measure leading whitespace
466 if line[pos] == ' ':
467 column += 1
468 elif line[pos] == '\t':
469 column = (column//tabsize + 1)*tabsize
470 elif line[pos] == '\f':
471 column = 0
472 else:
473 break
474 pos += 1
475 if pos == max:
476 break
477
478 if line[pos] in '#\r\n': # skip comments or blank lines
479 if line[pos] == '#':
480 comment_token = line[pos:].rstrip('\r\n')
481 nl_pos = pos + len(comment_token)
482 yield TokenInfo(COMMENT, comment_token,
483 (lnum, pos), (lnum, pos + len(comment_token)), line)
484 yield TokenInfo(NEWLINE, line[nl_pos:],
485 (lnum, nl_pos), (lnum, len(line)), line)
486 else:
487 yield TokenInfo(NEWLINE, line[pos:],
488 (lnum, pos), (lnum, len(line)), line)
489 continue
490
491 if column > indents[-1]: # count indents or dedents
492 indents.append(column)
493 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
494 while column < indents[-1]:
495 if column not in indents:
496 raise IndentationError(
497 "unindent does not match any outer indentation level",
498 ("<tokenize>", lnum, pos, line))
499 indents = indents[:-1]
500 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
501
502 else: # continued statement
503 if not line:
504 raise TokenError("EOF in multi-line statement", (lnum, 0))
505 continued = 0
506
507 while pos < max:
508 pseudomatch = pseudoprog.match(line, pos)
509 if pseudomatch: # scan for tokens
510 start, end = pseudomatch.span(1)
511 spos, epos, pos = (lnum, start), (lnum, end), end
512 token, initial = line[start:end], line[start]
513
514 if (initial in numchars or # ordinary number
515 (initial == '.' and token != '.' and token != '...')):
516 yield TokenInfo(NUMBER, token, spos, epos, line)
517 elif initial in '\r\n':
518 yield TokenInfo(NL if parenlev > 0 else NEWLINE,
519 token, spos, epos, line)
520 elif initial == '#':
521 assert not token.endswith("\n")
522 yield TokenInfo(COMMENT, token, spos, epos, line)
523 elif token in triple_quoted:
524 endprog = endprogs[token]
525 endmatch = endprog.match(line, pos)
526 if endmatch: # all on one line
527 pos = endmatch.end(0)
528 token = line[start:pos]
529 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
530 else:
531 strstart = (lnum, start) # multiple lines
532 contstr = line[start:]
533 contline = line
534 break
535 elif initial in single_quoted or \
536 token[:2] in single_quoted or \
537 token[:3] in single_quoted:
538 if token[-1] == '\n': # continued string
539 strstart = (lnum, start)
540 endprog = (endprogs[initial] or endprogs[token[1]] or
541 endprogs[token[2]])
542 contstr, needcont = line[start:], 1
543 contline = line
544 break
545 else: # ordinary string
546 yield TokenInfo(STRING, token, spos, epos, line)
547 elif initial.isidentifier(): # ordinary name
548 yield TokenInfo(NAME, token, spos, epos, line)
549 elif initial == '\\': # continued stmt
550 continued = 1
551 else:
552 if initial in '([{':
553 parenlev += 1
554 elif initial in ')]}':
555 parenlev -= 1
556 yield TokenInfo(OP, token, spos, epos, line)
557 else:
558 yield TokenInfo(ERRORTOKEN, line[pos],
559 (lnum, pos), (lnum, pos+1), line)
560 pos += 1
561
562 for indent in indents[1:]: # pop remaining indent levels
563 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
564 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
565
566
567 # An undocumented, backwards compatible, API for all the places in the standard
568 # library that expect to be able to use tokenize with strings
569 def generate_tokens(readline):
570 return _tokenize(readline, None)
571
572 if __name__ == "__main__":
573 # Quick sanity check
574 s = b'''def parseline(self, line):
575 """Parse the line into a command name and a string containing
576 the arguments. Returns a tuple containing (command, args, line).
577 'command' and 'args' may be None if the line couldn't be parsed.
578 """
579 line = line.strip()
580 if not line:
581 return None, None, line
582 elif line[0] == '?':
583 line = 'help ' + line[1:]
584 elif line[0] == '!':
585 if hasattr(self, 'do_shell'):
586 line = 'shell ' + line[1:]
587 else:
588 return None, None, line
589 i, n = 0, len(line)
590 while i < n and line[i] in self.identchars: i = i+1
591 cmd, arg = line[:i], line[i:].strip()
592 return cmd, arg, line
593 '''
594 for tok in tokenize(iter(s.splitlines()).__next__):
595 print(tok)
@@ -1,299 +1,297 b''
1 1 #!/usr/bin/env python
2 2 # -*- coding: utf-8 -*-
3 3 """Setup script for IPython.
4 4
5 5 Under Posix environments it works like a typical setup.py script.
6 6 Under Windows, the command sdist is not supported, since IPython
7 7 requires utilities which are not available under Windows."""
8 8
9 9 #-----------------------------------------------------------------------------
10 10 # Copyright (c) 2008-2011, IPython Development Team.
11 11 # Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
12 12 # Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
13 13 # Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
14 14 #
15 15 # Distributed under the terms of the Modified BSD License.
16 16 #
17 17 # The full license is in the file COPYING.rst, distributed with this software.
18 18 #-----------------------------------------------------------------------------
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Minimal Python version sanity check
22 22 #-----------------------------------------------------------------------------
23 23 from __future__ import print_function
24 24
25 25 import sys
26 26
27 27 # This check is also made in IPython/__init__, don't forget to update both when
28 28 # changing Python version requirements.
29 29 if sys.version_info < (3,3):
30 30 error = """
31 31 IPython 6.0+ does not support Python 2.6, 2.7, 3.0, 3.1, or 3.2.
32 32 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
33 33 Beginning with IPython 6.0, Python 3.3 and above is required.
34 34
35 35 See IPython `README.rst` file for more information:
36 36
37 37 https://github.com/ipython/ipython/blob/master/README.rst
38 38
39 39 """
40 40
41 41 print(error, file=sys.stderr)
42 42 sys.exit(1)
43 43
44 PY3 = (sys.version_info[0] >= 3)
45
46 44 # At least we're on the python version we need, move on.
47 45
48 46 #-------------------------------------------------------------------------------
49 47 # Imports
50 48 #-------------------------------------------------------------------------------
51 49
52 50 # Stdlib imports
53 51 import os
54 52
55 53 from glob import glob
56 54
57 55 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
58 56 # update it when the contents of directories change.
59 57 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
60 58
61 59 from distutils.core import setup
62 60
63 61 # Our own imports
64 62 from setupbase import target_update
65 63
66 64 from setupbase import (
67 65 setup_args,
68 66 find_packages,
69 67 find_package_data,
70 68 check_package_data_first,
71 69 find_entry_points,
72 70 build_scripts_entrypt,
73 71 find_data_files,
74 72 git_prebuild,
75 73 install_symlinked,
76 74 install_lib_symlink,
77 75 install_scripts_for_symlink,
78 76 unsymlink,
79 77 )
80 78
81 79 isfile = os.path.isfile
82 80 pjoin = os.path.join
83 81
84 82 #-------------------------------------------------------------------------------
85 83 # Handle OS specific things
86 84 #-------------------------------------------------------------------------------
87 85
88 86 if os.name in ('nt','dos'):
89 87 os_name = 'windows'
90 88 else:
91 89 os_name = os.name
92 90
93 91 # Under Windows, 'sdist' has not been supported. Now that the docs build with
94 92 # Sphinx it might work, but let's not turn it on until someone confirms that it
95 93 # actually works.
96 94 if os_name == 'windows' and 'sdist' in sys.argv:
97 95 print('The sdist command is not available under Windows. Exiting.')
98 96 sys.exit(1)
99 97
100 98
101 99 #-------------------------------------------------------------------------------
102 100 # Things related to the IPython documentation
103 101 #-------------------------------------------------------------------------------
104 102
105 103 # update the manuals when building a source dist
106 104 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
107 105
108 106 # List of things to be updated. Each entry is a triplet of args for
109 107 # target_update()
110 108 to_update = [
111 109 ('docs/man/ipython.1.gz',
112 110 ['docs/man/ipython.1'],
113 111 'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
114 112 ]
115 113
116 114
117 115 [ target_update(*t) for t in to_update ]
118 116
119 117 #---------------------------------------------------------------------------
120 118 # Find all the packages, package data, and data_files
121 119 #---------------------------------------------------------------------------
122 120
123 121 packages = find_packages()
124 122 package_data = find_package_data()
125 123
126 124 data_files = find_data_files()
127 125
128 126 setup_args['packages'] = packages
129 127 setup_args['package_data'] = package_data
130 128 setup_args['data_files'] = data_files
131 129
132 130 #---------------------------------------------------------------------------
133 131 # custom distutils commands
134 132 #---------------------------------------------------------------------------
135 133 # imports here, so they are after setuptools import if there was one
136 134 from distutils.command.sdist import sdist
137 135 from distutils.command.upload import upload
138 136
139 137 class UploadWindowsInstallers(upload):
140 138
141 139 description = "Upload Windows installers to PyPI (only used from tools/release_windows.py)"
142 140 user_options = upload.user_options + [
143 141 ('files=', 'f', 'exe file (or glob) to upload')
144 142 ]
145 143 def initialize_options(self):
146 144 upload.initialize_options(self)
147 145 meta = self.distribution.metadata
148 146 base = '{name}-{version}'.format(
149 147 name=meta.get_name(),
150 148 version=meta.get_version()
151 149 )
152 150 self.files = os.path.join('dist', '%s.*.exe' % base)
153 151
154 152 def run(self):
155 153 for dist_file in glob(self.files):
156 154 self.upload_file('bdist_wininst', 'any', dist_file)
157 155
158 156 setup_args['cmdclass'] = {
159 157 'build_py': \
160 158 check_package_data_first(git_prebuild('IPython')),
161 159 'sdist' : git_prebuild('IPython', sdist),
162 160 'upload_wininst' : UploadWindowsInstallers,
163 161 'symlink': install_symlinked,
164 162 'install_lib_symlink': install_lib_symlink,
165 163 'install_scripts_sym': install_scripts_for_symlink,
166 164 'unsymlink': unsymlink,
167 165 }
168 166
169 167
170 168 #---------------------------------------------------------------------------
171 169 # Handle scripts, dependencies, and setuptools specific things
172 170 #---------------------------------------------------------------------------
173 171
174 172 # For some commands, use setuptools. Note that we do NOT list install here!
175 173 # If you want a setuptools-enhanced install, just run 'setupegg.py install'
176 174 needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm',
177 175 'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel',
178 176 'egg_info', 'easy_install', 'upload', 'install_egg_info',
179 177 ))
180 178
181 179 if len(needs_setuptools.intersection(sys.argv)) > 0:
182 180 import setuptools
183 181
184 182 # This dict is used for passing extra arguments that are setuptools
185 183 # specific to setup
186 184 setuptools_extra_args = {}
187 185
188 186 # setuptools requirements
189 187
190 188 extras_require = dict(
191 189 parallel = ['ipyparallel'],
192 190 qtconsole = ['qtconsole'],
193 191 doc = ['Sphinx>=1.3'],
194 192 test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy'],
195 193 terminal = [],
196 194 kernel = ['ipykernel'],
197 195 nbformat = ['nbformat'],
198 196 notebook = ['notebook', 'ipywidgets'],
199 197 nbconvert = ['nbconvert'],
200 198 )
201 199
202 200 install_requires = [
203 201 'setuptools>=18.5',
204 202 'decorator',
205 203 'pickleshare',
206 204 'simplegeneric>0.8',
207 205 'traitlets>=4.2',
208 206 'prompt_toolkit>=1.0.3,<2.0.0',
209 207 'pygments',
210 208 ]
211 209
212 210 # Platform-specific dependencies:
213 211 # This is the correct way to specify these,
214 212 # but requires pip >= 6. pip < 6 ignores these.
215 213
216 214 extras_require.update({
217 215 ':python_version == "2.7"': ['backports.shutil_get_terminal_size'],
218 216 ':python_version == "2.7" or python_version == "3.3"': ['pathlib2'],
219 217 ':sys_platform != "win32"': ['pexpect'],
220 218 ':sys_platform == "darwin"': ['appnope'],
221 219 ':sys_platform == "win32"': ['colorama'],
222 220 ':sys_platform == "win32" and python_version < "3.6"': ['win_unicode_console>=0.5'],
223 221 'test:python_version == "2.7"': ['mock'],
224 222 })
225 223 # FIXME: re-specify above platform dependencies for pip < 6
226 224 # These would result in non-portable bdists.
227 225 if not any(arg.startswith('bdist') for arg in sys.argv):
228 226 if sys.version_info < (3, 3):
229 227 extras_require['test'].append('mock')
230 228
231 229 if sys.platform == 'darwin':
232 230 install_requires.extend(['appnope'])
233 231
234 232 if not sys.platform.startswith('win'):
235 233 install_requires.append('pexpect')
236 234
237 235 # workaround pypa/setuptools#147, where setuptools misspells
238 236 # platform_python_implementation as python_implementation
239 237 if 'setuptools' in sys.modules:
240 238 for key in list(extras_require):
241 239 if 'platform_python_implementation' in key:
242 240 new_key = key.replace('platform_python_implementation', 'python_implementation')
243 241 extras_require[new_key] = extras_require.pop(key)
244 242
245 243 everything = set()
246 244 for key, deps in extras_require.items():
247 245 if ':' not in key:
248 246 everything.update(deps)
249 247 extras_require['all'] = everything
250 248
251 249 if 'setuptools' in sys.modules:
252 250 setuptools_extra_args['python_requires'] = '>=3.3'
253 251 setuptools_extra_args['zip_safe'] = False
254 252 setuptools_extra_args['entry_points'] = {
255 253 'console_scripts': find_entry_points(),
256 254 'pygments.lexers': [
257 255 'ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer',
258 256 'ipython = IPython.lib.lexers:IPythonLexer',
259 257 'ipython3 = IPython.lib.lexers:IPython3Lexer',
260 258 ],
261 259 }
262 260 setup_args['extras_require'] = extras_require
263 261 requires = setup_args['install_requires'] = install_requires
264 262
265 263 # Script to be run by the windows binary installer after the default setup
266 264 # routine, to add shortcuts and similar windows-only things. Windows
267 265 # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
268 266 # doesn't find them.
269 267 if 'bdist_wininst' in sys.argv:
270 268 if len(sys.argv) > 2 and \
271 269 ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
272 270 print("ERROR: bdist_wininst must be run alone. Exiting.", file=sys.stderr)
273 271 sys.exit(1)
274 272 setup_args['data_files'].append(
275 273 ['Scripts', ('scripts/ipython.ico', 'scripts/ipython_nb.ico')])
276 274 setup_args['scripts'] = [pjoin('scripts','ipython_win_post_install.py')]
277 275 setup_args['options'] = {"bdist_wininst":
278 276 {"install_script":
279 277 "ipython_win_post_install.py"}}
280 278
281 279 else:
282 280 # scripts has to be a non-empty list, or install_scripts isn't called
283 281 setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()]
284 282
285 283 setup_args['cmdclass']['build_scripts'] = build_scripts_entrypt
286 284
287 285 #---------------------------------------------------------------------------
288 286 # Do the actual setup now
289 287 #---------------------------------------------------------------------------
290 288
291 289 setup_args.update(setuptools_extra_args)
292 290
293 291
294 292
295 293 def main():
296 294 setup(**setup_args)
297 295
298 296 if __name__ == '__main__':
299 297 main()
@@ -1,235 +1,232 b''
1 1 #!/usr/bin/env python
2 2 """Simple tools to query github.com and gather stats about issues.
3 3
4 4 To generate a report for IPython 2.0, run:
5 5
6 6 python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
7 7 """
8 8 #-----------------------------------------------------------------------------
9 9 # Imports
10 10 #-----------------------------------------------------------------------------
11 11
12 12 from __future__ import print_function
13 13
14 14 import codecs
15 15 import sys
16 16
17 17 from argparse import ArgumentParser
18 18 from datetime import datetime, timedelta
19 19 from subprocess import check_output
20 20
21 21 from gh_api import (
22 22 get_paged_request, make_auth_header, get_pull_request, is_pull_request,
23 23 get_milestone_id, get_issues_list, get_authors,
24 24 )
25 25 #-----------------------------------------------------------------------------
26 26 # Globals
27 27 #-----------------------------------------------------------------------------
28 28
29 29 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
30 30 PER_PAGE = 100
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Functions
34 34 #-----------------------------------------------------------------------------
35 35
36 36 def round_hour(dt):
37 37 return dt.replace(minute=0,second=0,microsecond=0)
38 38
39 39 def _parse_datetime(s):
40 40 """Parse dates in the format returned by the Github API."""
41 41 if s:
42 42 return datetime.strptime(s, ISO8601)
43 43 else:
44 44 return datetime.fromtimestamp(0)
45 45
46 46 def issues2dict(issues):
47 47 """Convert a list of issues to a dict, keyed by issue number."""
48 48 idict = {}
49 49 for i in issues:
50 50 idict[i['number']] = i
51 51 return idict
52 52
53 53 def split_pulls(all_issues, project="ipython/ipython"):
54 54 """split a list of closed issues into non-PR Issues and Pull Requests"""
55 55 pulls = []
56 56 issues = []
57 57 for i in all_issues:
58 58 if is_pull_request(i):
59 59 pull = get_pull_request(project, i['number'], auth=True)
60 60 pulls.append(pull)
61 61 else:
62 62 issues.append(i)
63 63 return issues, pulls
64 64
65 65
66 66 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
67 67 """Get all issues closed since a particular point in time. period
68 68 can either be a datetime object, or a timedelta object. In the
69 69 latter case, it is used as a time before the present.
70 70 """
71 71
72 72 which = 'pulls' if pulls else 'issues'
73 73
74 74 if isinstance(period, timedelta):
75 75 since = round_hour(datetime.utcnow() - period)
76 76 else:
77 77 since = period
78 78 url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
79 79 allclosed = get_paged_request(url, headers=make_auth_header())
80 80
81 81 filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
82 82 if pulls:
83 83 filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
84 84 # filter out PRs not against master (backports)
85 85 filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
86 86 else:
87 87 filtered = [ i for i in filtered if not is_pull_request(i) ]
88 88
89 89 return filtered
90 90
91 91
92 92 def sorted_by_field(issues, field='closed_at', reverse=False):
93 93 """Return a list of issues sorted by closing date date."""
94 94 return sorted(issues, key = lambda i:i[field], reverse=reverse)
95 95
96 96
97 97 def report(issues, show_urls=False):
98 98 """Summary report about a list of issues, printing number and title."""
99 99 if show_urls:
100 100 for i in issues:
101 101 role = 'ghpull' if 'merged_at' in i else 'ghissue'
102 102 print(u'* :%s:`%d`: %s' % (role, i['number'],
103 103 i['title'].replace(u'`', u'``')))
104 104 else:
105 105 for i in issues:
106 106 print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
107 107
108 108 #-----------------------------------------------------------------------------
109 109 # Main script
110 110 #-----------------------------------------------------------------------------
111 111
112 112 if __name__ == "__main__":
113 113
114 114 print("DEPRECATE: backport_pr.py is deprecated and is is now recommended"
115 115 "to install `ghpro` from PyPI.", file=sys.stderr)
116 116
117 # deal with unicode
118 if sys.version_info < (3,):
119 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
120 117
121 118 # Whether to add reST urls for all issues in printout.
122 119 show_urls = True
123 120
124 121 parser = ArgumentParser()
125 122 parser.add_argument('--since-tag', type=str,
126 123 help="The git tag to use for the starting point (typically the last major release)."
127 124 )
128 125 parser.add_argument('--milestone', type=str,
129 126 help="The GitHub milestone to use for filtering issues [optional]."
130 127 )
131 128 parser.add_argument('--days', type=int,
132 129 help="The number of days of data to summarize (use this or --since-tag)."
133 130 )
134 131 parser.add_argument('--project', type=str, default="ipython/ipython",
135 132 help="The project to summarize."
136 133 )
137 134 parser.add_argument('--links', action='store_true', default=False,
138 135 help="Include links to all closed Issues and PRs in the output."
139 136 )
140 137
141 138 opts = parser.parse_args()
142 139 tag = opts.since_tag
143 140
144 141 # set `since` from days or git tag
145 142 if opts.days:
146 143 since = datetime.utcnow() - timedelta(days=opts.days)
147 144 else:
148 145 if not tag:
149 146 tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
150 147 cmd = ['git', 'log', '-1', '--format=%ai', tag]
151 148 tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
152 149 since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
153 150 h = int(tz[1:3])
154 151 m = int(tz[3:])
155 152 td = timedelta(hours=h, minutes=m)
156 153 if tz[0] == '-':
157 154 since += td
158 155 else:
159 156 since -= td
160 157
161 158 since = round_hour(since)
162 159
163 160 milestone = opts.milestone
164 161 project = opts.project
165 162
166 163 print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
167 164 if milestone:
168 165 milestone_id = get_milestone_id(project=project, milestone=milestone,
169 166 auth=True)
170 167 issues_and_pulls = get_issues_list(project=project,
171 168 milestone=milestone_id,
172 169 state='closed',
173 170 auth=True,
174 171 )
175 172 issues, pulls = split_pulls(issues_and_pulls, project=project)
176 173 else:
177 174 issues = issues_closed_since(since, project=project, pulls=False)
178 175 pulls = issues_closed_since(since, project=project, pulls=True)
179 176
180 177 # For regular reports, it's nice to show them in reverse chronological order
181 178 issues = sorted_by_field(issues, reverse=True)
182 179 pulls = sorted_by_field(pulls, reverse=True)
183 180
184 181 n_issues, n_pulls = map(len, (issues, pulls))
185 182 n_total = n_issues + n_pulls
186 183
187 184 # Print summary report we can directly include into release notes.
188 185
189 186 print()
190 187 since_day = since.strftime("%Y/%m/%d")
191 188 today = datetime.today().strftime("%Y/%m/%d")
192 189 print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
193 190 print()
194 191 print("These lists are automatically generated, and may be incomplete or contain duplicates.")
195 192 print()
196 193
197 194 ncommits = 0
198 195 all_authors = []
199 196 if tag:
200 197 # print git info, in addition to GitHub info:
201 198 since_tag = tag+'..'
202 199 cmd = ['git', 'log', '--oneline', since_tag]
203 200 ncommits += len(check_output(cmd).splitlines())
204 201
205 202 author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
206 203 all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
207 204
208 205 pr_authors = []
209 206 for pr in pulls:
210 207 pr_authors.extend(get_authors(pr))
211 208 ncommits = len(pr_authors) + ncommits - len(pulls)
212 209 author_cmd = ['git', 'check-mailmap'] + pr_authors
213 210 with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
214 211 all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
215 212 unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
216 213
217 214 print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
218 215 if milestone:
219 216 print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}+>`__".format(project=project,milestone=milestone)
220 217 )
221 218
222 219 print()
223 220 print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
224 221 print()
225 222 print('\n'.join(unique_authors))
226 223
227 224 if opts.links:
228 225 print()
229 226 print("GitHub issues and pull requests:")
230 227 print()
231 228 print('Pull Requests (%d):\n' % n_pulls)
232 229 report(pulls, show_urls)
233 230 print()
234 231 print('Issues (%d):\n' % n_issues)
235 232 report(issues, show_urls)
@@ -1,439 +0,0 b''
1 """Patched version of standard library tokenize, to deal with various bugs.
2
3 Patches
4
5 - Relevant parts of Gareth Rees' patch for Python issue #12691 (untokenizing),
6 manually applied.
7 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
8 on whether they are in a multi-line statement. Filed as Python issue #17061.
9
10 -------------------------------------------------------------------------------
11 Tokenization help for Python programs.
12
13 generate_tokens(readline) is a generator that breaks a stream of
14 text into Python tokens. It accepts a readline-like method which is called
15 repeatedly to get the next line of input (or "" for EOF). It generates
16 5-tuples with these members:
17
18 the token type (see token.py)
19 the token (a string)
20 the starting (row, column) indices of the token (a 2-tuple of ints)
21 the ending (row, column) indices of the token (a 2-tuple of ints)
22 the original line (string)
23
24 It is designed to match the working of the Python tokenizer exactly, except
25 that it produces COMMENT tokens for comments and gives type OP for all
26 operators
27
28 Older entry points
29 tokenize_loop(readline, tokeneater)
30 tokenize(readline, tokeneater=printtoken)
31 are the same, except instead of generating tokens, tokeneater is a callback
32 function to which the 5 fields described above are passed as 5 arguments,
33 each time a new token is found."""
34 from __future__ import print_function
35
36 __author__ = 'Ka-Ping Yee <ping@lfw.org>'
37 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
38 'Skip Montanaro, Raymond Hettinger')
39
40 import string, re
41 from token import *
42
43 import token
44 __all__ = [x for x in dir(token) if not x.startswith("_")]
45 __all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
46 del x
47 del token
48
49 __all__ += ["TokenError"]
50
51 COMMENT = N_TOKENS
52 tok_name[COMMENT] = 'COMMENT'
53 NL = N_TOKENS + 1
54 tok_name[NL] = 'NL'
55 N_TOKENS += 2
56
57 def group(*choices): return '(' + '|'.join(choices) + ')'
58 def any(*choices): return group(*choices) + '*'
59 def maybe(*choices): return group(*choices) + '?'
60
61 Whitespace = r'[ \f\t]*'
62 Comment = r'#[^\r\n]*'
63 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
64 Name = r'[a-zA-Z_]\w*'
65
66 Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
67 Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
68 Binnumber = r'0[bB][01]+[lL]?'
69 Decnumber = r'[1-9]\d*[lL]?'
70 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
71 Exponent = r'[eE][-+]?\d+'
72 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
73 Expfloat = r'\d+' + Exponent
74 Floatnumber = group(Pointfloat, Expfloat)
75 Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
76 Number = group(Imagnumber, Floatnumber, Intnumber)
77
78 # Tail end of ' string.
79 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
80 # Tail end of " string.
81 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
82 # Tail end of ''' string.
83 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
84 # Tail end of """ string.
85 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
86 Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
87 # Single-line ' or " string.
88 String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
89 r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
90
91 # Because of leftmost-then-longest match semantics, be sure to put the
92 # longest operators first (e.g., if = came before ==, == would get
93 # recognized as two instances of =).
94 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
95 r"//=?",
96 r"[+\-*/%&|^=<>]=?",
97 r"~")
98
99 Bracket = '[][(){}]'
100 Special = group(r'\r?\n', r'[:;.,`@]')
101 Funny = group(Operator, Bracket, Special)
102
103 PlainToken = group(Number, Funny, String, Name)
104 Token = Ignore + PlainToken
105
106 # First (or only) line of ' or " string.
107 ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
108 group("'", r'\\\r?\n'),
109 r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
110 group('"', r'\\\r?\n'))
111 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
112 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
113
114 tokenprog, pseudoprog, single3prog, double3prog = map(
115 re.compile, (Token, PseudoToken, Single3, Double3))
116 endprogs = {"'": re.compile(Single), '"': re.compile(Double),
117 "'''": single3prog, '"""': double3prog,
118 "r'''": single3prog, 'r"""': double3prog,
119 "u'''": single3prog, 'u"""': double3prog,
120 "ur'''": single3prog, 'ur"""': double3prog,
121 "R'''": single3prog, 'R"""': double3prog,
122 "U'''": single3prog, 'U"""': double3prog,
123 "uR'''": single3prog, 'uR"""': double3prog,
124 "Ur'''": single3prog, 'Ur"""': double3prog,
125 "UR'''": single3prog, 'UR"""': double3prog,
126 "b'''": single3prog, 'b"""': double3prog,
127 "br'''": single3prog, 'br"""': double3prog,
128 "B'''": single3prog, 'B"""': double3prog,
129 "bR'''": single3prog, 'bR"""': double3prog,
130 "Br'''": single3prog, 'Br"""': double3prog,
131 "BR'''": single3prog, 'BR"""': double3prog,
132 'r': None, 'R': None, 'u': None, 'U': None,
133 'b': None, 'B': None}
134
135 triple_quoted = {}
136 for t in ("'''", '"""',
137 "r'''", 'r"""', "R'''", 'R"""',
138 "u'''", 'u"""', "U'''", 'U"""',
139 "ur'''", 'ur"""', "Ur'''", 'Ur"""',
140 "uR'''", 'uR"""', "UR'''", 'UR"""',
141 "b'''", 'b"""', "B'''", 'B"""',
142 "br'''", 'br"""', "Br'''", 'Br"""',
143 "bR'''", 'bR"""', "BR'''", 'BR"""'):
144 triple_quoted[t] = t
145 single_quoted = {}
146 for t in ("'", '"',
147 "r'", 'r"', "R'", 'R"',
148 "u'", 'u"', "U'", 'U"',
149 "ur'", 'ur"', "Ur'", 'Ur"',
150 "uR'", 'uR"', "UR'", 'UR"',
151 "b'", 'b"', "B'", 'B"',
152 "br'", 'br"', "Br'", 'Br"',
153 "bR'", 'bR"', "BR'", 'BR"' ):
154 single_quoted[t] = t
155
156 tabsize = 8
157
158 class TokenError(Exception): pass
159
160 class StopTokenizing(Exception): pass
161
162 def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
163 srow, scol = srow_scol
164 erow, ecol = erow_ecol
165 print("%d,%d-%d,%d:\t%s\t%s" % \
166 (srow, scol, erow, ecol, tok_name[type], repr(token)))
167
168 def tokenize(readline, tokeneater=printtoken):
169 """
170 The tokenize() function accepts two parameters: one representing the
171 input stream, and one providing an output mechanism for tokenize().
172
173 The first parameter, readline, must be a callable object which provides
174 the same interface as the readline() method of built-in file objects.
175 Each call to the function should return one line of input as a string.
176
177 The second parameter, tokeneater, must also be a callable object. It is
178 called once for each token, with five arguments, corresponding to the
179 tuples generated by generate_tokens().
180 """
181 try:
182 tokenize_loop(readline, tokeneater)
183 except StopTokenizing:
184 pass
185
186 # backwards compatible interface
187 def tokenize_loop(readline, tokeneater):
188 for token_info in generate_tokens(readline):
189 tokeneater(*token_info)
190
191 class Untokenizer:
192
193 def __init__(self):
194 self.tokens = []
195 self.prev_row = 1
196 self.prev_col = 0
197
198 def add_whitespace(self, start):
199 row, col = start
200 assert row >= self.prev_row
201 col_offset = col - self.prev_col
202 if col_offset > 0:
203 self.tokens.append(" " * col_offset)
204 elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
205 # Line was backslash-continued
206 self.tokens.append(" ")
207
208 def untokenize(self, tokens):
209 iterable = iter(tokens)
210 for t in iterable:
211 if len(t) == 2:
212 self.compat(t, iterable)
213 break
214 tok_type, token, start, end = t[:4]
215 self.add_whitespace(start)
216 self.tokens.append(token)
217 self.prev_row, self.prev_col = end
218 if tok_type in (NEWLINE, NL):
219 self.prev_row += 1
220 self.prev_col = 0
221 return "".join(self.tokens)
222
223 def compat(self, token, iterable):
224 # This import is here to avoid problems when the itertools
225 # module is not built yet and tokenize is imported.
226 from itertools import chain
227 startline = False
228 prevstring = False
229 indents = []
230 toks_append = self.tokens.append
231 for tok in chain([token], iterable):
232 toknum, tokval = tok[:2]
233
234 if toknum in (NAME, NUMBER):
235 tokval += ' '
236
237 # Insert a space between two consecutive strings
238 if toknum == STRING:
239 if prevstring:
240 tokval = ' ' + tokval
241 prevstring = True
242 else:
243 prevstring = False
244
245 if toknum == INDENT:
246 indents.append(tokval)
247 continue
248 elif toknum == DEDENT:
249 indents.pop()
250 continue
251 elif toknum in (NEWLINE, NL):
252 startline = True
253 elif startline and indents:
254 toks_append(indents[-1])
255 startline = False
256 toks_append(tokval)
257
258 def untokenize(iterable):
259 """Transform tokens back into Python source code.
260
261 Each element returned by the iterable must be a token sequence
262 with at least two elements, a token number and token value. If
263 only two tokens are passed, the resulting output is poor.
264
265 Round-trip invariant for full input:
266 Untokenized source will match input source exactly
267
268 Round-trip invariant for limited intput:
269 # Output text will tokenize the back to the input
270 t1 = [tok[:2] for tok in generate_tokens(f.readline)]
271 newcode = untokenize(t1)
272 readline = iter(newcode.splitlines(1)).next
273 t2 = [tok[:2] for tok in generate_tokens(readline)]
274 assert t1 == t2
275 """
276 ut = Untokenizer()
277 return ut.untokenize(iterable)
278
279 def generate_tokens(readline):
280 """
281 The generate_tokens() generator requires one argment, readline, which
282 must be a callable object which provides the same interface as the
283 readline() method of built-in file objects. Each call to the function
284 should return one line of input as a string. Alternately, readline
285 can be a callable function terminating with StopIteration:
286 readline = open(myfile).next # Example of alternate readline
287
288 The generator produces 5-tuples with these members: the token type; the
289 token string; a 2-tuple (srow, scol) of ints specifying the row and
290 column where the token begins in the source; a 2-tuple (erow, ecol) of
291 ints specifying the row and column where the token ends in the source;
292 and the line on which the token was found. The line passed is the
293 logical line; continuation lines are included.
294 """
295 lnum = parenlev = continued = 0
296 namechars, numchars = string.ascii_letters + '_', '0123456789'
297 contstr, needcont = '', 0
298 contline = None
299 indents = [0]
300
301 while 1: # loop over lines in stream
302 try:
303 line = readline()
304 except StopIteration:
305 line = ''
306 lnum += 1
307 pos, max = 0, len(line)
308
309 if contstr: # continued string
310 if not line:
311 raise TokenError("EOF in multi-line string", strstart)
312 endmatch = endprog.match(line)
313 if endmatch:
314 pos = end = endmatch.end(0)
315 yield (STRING, contstr + line[:end],
316 strstart, (lnum, end), contline + line)
317 contstr, needcont = '', 0
318 contline = None
319 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
320 yield (ERRORTOKEN, contstr + line,
321 strstart, (lnum, len(line)), contline)
322 contstr = ''
323 contline = None
324 continue
325 else:
326 contstr = contstr + line
327 contline = contline + line
328 continue
329
330 elif parenlev == 0 and not continued: # new statement
331 if not line: break
332 column = 0
333 while pos < max: # measure leading whitespace
334 if line[pos] == ' ':
335 column += 1
336 elif line[pos] == '\t':
337 column = (column//tabsize + 1)*tabsize
338 elif line[pos] == '\f':
339 column = 0
340 else:
341 break
342 pos += 1
343 if pos == max:
344 break
345
346 if line[pos] in '#\r\n': # skip comments or blank lines
347 if line[pos] == '#':
348 comment_token = line[pos:].rstrip('\r\n')
349 nl_pos = pos + len(comment_token)
350 yield (COMMENT, comment_token,
351 (lnum, pos), (lnum, pos + len(comment_token)), line)
352 yield (NEWLINE, line[nl_pos:],
353 (lnum, nl_pos), (lnum, len(line)), line)
354 else:
355 yield (NEWLINE, line[pos:],
356 (lnum, pos), (lnum, len(line)), line)
357 continue
358
359 if column > indents[-1]: # count indents or dedents
360 indents.append(column)
361 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
362 while column < indents[-1]:
363 if column not in indents:
364 raise IndentationError(
365 "unindent does not match any outer indentation level",
366 ("<tokenize>", lnum, pos, line))
367 indents = indents[:-1]
368 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
369
370 else: # continued statement
371 if not line:
372 raise TokenError("EOF in multi-line statement", (lnum, 0))
373 continued = 0
374
375 while pos < max:
376 pseudomatch = pseudoprog.match(line, pos)
377 if pseudomatch: # scan for tokens
378 start, end = pseudomatch.span(1)
379 spos, epos, pos = (lnum, start), (lnum, end), end
380 token, initial = line[start:end], line[start]
381
382 if initial in numchars or \
383 (initial == '.' and token != '.'): # ordinary number
384 yield (NUMBER, token, spos, epos, line)
385 elif initial in '\r\n':
386 yield (NL if parenlev > 0 else NEWLINE,
387 token, spos, epos, line)
388 elif initial == '#':
389 assert not token.endswith("\n")
390 yield (COMMENT, token, spos, epos, line)
391 elif token in triple_quoted:
392 endprog = endprogs[token]
393 endmatch = endprog.match(line, pos)
394 if endmatch: # all on one line
395 pos = endmatch.end(0)
396 token = line[start:pos]
397 yield (STRING, token, spos, (lnum, pos), line)
398 else:
399 strstart = (lnum, start) # multiple lines
400 contstr = line[start:]
401 contline = line
402 break
403 elif initial in single_quoted or \
404 token[:2] in single_quoted or \
405 token[:3] in single_quoted:
406 if token[-1] == '\n': # continued string
407 strstart = (lnum, start)
408 endprog = (endprogs[initial] or endprogs[token[1]] or
409 endprogs[token[2]])
410 contstr, needcont = line[start:], 1
411 contline = line
412 break
413 else: # ordinary string
414 yield (STRING, token, spos, epos, line)
415 elif initial in namechars: # ordinary name
416 yield (NAME, token, spos, epos, line)
417 elif initial == '\\': # continued stmt
418 continued = 1
419 else:
420 if initial in '([{':
421 parenlev += 1
422 elif initial in ')]}':
423 parenlev -= 1
424 yield (OP, token, spos, epos, line)
425 else:
426 yield (ERRORTOKEN, line[pos],
427 (lnum, pos), (lnum, pos+1), line)
428 pos += 1
429
430 for indent in indents[1:]: # pop remaining indent levels
431 yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
432 yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
433
434 if __name__ == '__main__': # testing
435 import sys
436 if len(sys.argv) > 1:
437 tokenize(open(sys.argv[1]).readline)
438 else:
439 tokenize(sys.stdin.readline)
This diff has been collapsed as it changes many lines, (595 lines changed) Show them Hide them
@@ -1,595 +0,0 b''
1 """Patched version of standard library tokenize, to deal with various bugs.
2
3 Based on Python 3.2 code.
4
5 Patches:
6
7 - Gareth Rees' patch for Python issue #12691 (untokenizing)
8 - Except we don't encode the output of untokenize
9 - Python 2 compatible syntax, so that it can be byte-compiled at installation
10 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
11 on whether they are in a multi-line statement. Filed as Python issue #17061.
12 - Export generate_tokens & TokenError
13 - u and rb literals are allowed under Python 3.3 and above.
14
15 ------------------------------------------------------------------------------
16 Tokenization help for Python programs.
17
18 tokenize(readline) is a generator that breaks a stream of bytes into
19 Python tokens. It decodes the bytes according to PEP-0263 for
20 determining source file encoding.
21
22 It accepts a readline-like method which is called repeatedly to get the
23 next line of input (or b"" for EOF). It generates 5-tuples with these
24 members:
25
26 the token type (see token.py)
27 the token (a string)
28 the starting (row, column) indices of the token (a 2-tuple of ints)
29 the ending (row, column) indices of the token (a 2-tuple of ints)
30 the original line (string)
31
32 It is designed to match the working of the Python tokenizer exactly, except
33 that it produces COMMENT tokens for comments and gives type OP for all
34 operators. Additionally, all token lists start with an ENCODING token
35 which tells you which encoding was used to decode the bytes stream.
36 """
37 from __future__ import absolute_import
38
39 __author__ = 'Ka-Ping Yee <ping@lfw.org>'
40 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
41 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
42 'Michael Foord')
43 import builtins
44 import re
45 import sys
46 from token import *
47 from codecs import lookup, BOM_UTF8
48 import collections
49 from io import TextIOWrapper
50 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
51
52 import token
53 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
54 "NL", "untokenize", "ENCODING", "TokenInfo"]
55 del token
56
57 __all__ += ["generate_tokens", "TokenError"]
58
59 COMMENT = N_TOKENS
60 tok_name[COMMENT] = 'COMMENT'
61 NL = N_TOKENS + 1
62 tok_name[NL] = 'NL'
63 ENCODING = N_TOKENS + 2
64 tok_name[ENCODING] = 'ENCODING'
65 N_TOKENS += 3
66
67 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
68 def __repr__(self):
69 annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
70 return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
71 self._replace(type=annotated_type))
72
73 def group(*choices): return '(' + '|'.join(choices) + ')'
74 def any(*choices): return group(*choices) + '*'
75 def maybe(*choices): return group(*choices) + '?'
76
77 # Note: we use unicode matching for names ("\w") but ascii matching for
78 # number literals.
79 Whitespace = r'[ \f\t]*'
80 Comment = r'#[^\r\n]*'
81 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
82 Name = r'\w+'
83
84 Hexnumber = r'0[xX][0-9a-fA-F]+'
85 Binnumber = r'0[bB][01]+'
86 Octnumber = r'0[oO][0-7]+'
87 Decnumber = r'(?:0+|[1-9][0-9]*)'
88 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
89 Exponent = r'[eE][-+]?[0-9]+'
90 Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
91 Expfloat = r'[0-9]+' + Exponent
92 Floatnumber = group(Pointfloat, Expfloat)
93 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
94 Number = group(Imagnumber, Floatnumber, Intnumber)
95
96 if sys.version_info.minor >= 3:
97 StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
98 else:
99 StringPrefix = r'(?:[bB]?[rR]?)?'
100
101 # Tail end of ' string.
102 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
103 # Tail end of " string.
104 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
105 # Tail end of ''' string.
106 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
107 # Tail end of """ string.
108 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
109 Triple = group(StringPrefix + "'''", StringPrefix + '"""')
110 # Single-line ' or " string.
111 String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
112 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
113
114 # Because of leftmost-then-longest match semantics, be sure to put the
115 # longest operators first (e.g., if = came before ==, == would get
116 # recognized as two instances of =).
117 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
118 r"//=?", r"->",
119 r"[+\-*/%&|^=<>]=?",
120 r"~")
121
122 Bracket = '[][(){}]'
123 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
124 Funny = group(Operator, Bracket, Special)
125
126 PlainToken = group(Number, Funny, String, Name)
127 Token = Ignore + PlainToken
128
129 # First (or only) line of ' or " string.
130 ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
131 group("'", r'\\\r?\n'),
132 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
133 group('"', r'\\\r?\n'))
134 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
135 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
136
137 def _compile(expr):
138 return re.compile(expr, re.UNICODE)
139
140 tokenprog, pseudoprog, single3prog, double3prog = map(
141 _compile, (Token, PseudoToken, Single3, Double3))
142 endprogs = {"'": _compile(Single), '"': _compile(Double),
143 "'''": single3prog, '"""': double3prog,
144 "r'''": single3prog, 'r"""': double3prog,
145 "b'''": single3prog, 'b"""': double3prog,
146 "R'''": single3prog, 'R"""': double3prog,
147 "B'''": single3prog, 'B"""': double3prog,
148 "br'''": single3prog, 'br"""': double3prog,
149 "bR'''": single3prog, 'bR"""': double3prog,
150 "Br'''": single3prog, 'Br"""': double3prog,
151 "BR'''": single3prog, 'BR"""': double3prog,
152 'r': None, 'R': None, 'b': None, 'B': None}
153
154 triple_quoted = {}
155 for t in ("'''", '"""',
156 "r'''", 'r"""', "R'''", 'R"""',
157 "b'''", 'b"""', "B'''", 'B"""',
158 "br'''", 'br"""', "Br'''", 'Br"""',
159 "bR'''", 'bR"""', "BR'''", 'BR"""'):
160 triple_quoted[t] = t
161 single_quoted = {}
162 for t in ("'", '"',
163 "r'", 'r"', "R'", 'R"',
164 "b'", 'b"', "B'", 'B"',
165 "br'", 'br"', "Br'", 'Br"',
166 "bR'", 'bR"', "BR'", 'BR"' ):
167 single_quoted[t] = t
168
169 if sys.version_info.minor >= 3:
170 # Python 3.3
171 for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
172 _t2 = _prefix+'"""'
173 endprogs[_t2] = double3prog
174 triple_quoted[_t2] = _t2
175 _t1 = _prefix + "'''"
176 endprogs[_t1] = single3prog
177 triple_quoted[_t1] = _t1
178 single_quoted[_prefix+'"'] = _prefix+'"'
179 single_quoted[_prefix+"'"] = _prefix+"'"
180 del _prefix, _t2, _t1
181 endprogs['u'] = None
182 endprogs['U'] = None
183
184 del _compile
185
186 tabsize = 8
187
188 class TokenError(Exception): pass
189
190 class StopTokenizing(Exception): pass
191
192
193 class Untokenizer:
194
195 def __init__(self):
196 self.tokens = []
197 self.prev_row = 1
198 self.prev_col = 0
199 self.encoding = 'utf-8'
200
201 def add_whitespace(self, tok_type, start):
202 row, col = start
203 assert row >= self.prev_row
204 col_offset = col - self.prev_col
205 if col_offset > 0:
206 self.tokens.append(" " * col_offset)
207 elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
208 # Line was backslash-continued.
209 self.tokens.append(" ")
210
211 def untokenize(self, tokens):
212 iterable = iter(tokens)
213 for t in iterable:
214 if len(t) == 2:
215 self.compat(t, iterable)
216 break
217 tok_type, token, start, end = t[:4]
218 if tok_type == ENCODING:
219 self.encoding = token
220 continue
221 self.add_whitespace(tok_type, start)
222 self.tokens.append(token)
223 self.prev_row, self.prev_col = end
224 if tok_type in (NEWLINE, NL):
225 self.prev_row += 1
226 self.prev_col = 0
227 return "".join(self.tokens)
228
229 def compat(self, token, iterable):
230 # This import is here to avoid problems when the itertools
231 # module is not built yet and tokenize is imported.
232 from itertools import chain
233 startline = False
234 prevstring = False
235 indents = []
236 toks_append = self.tokens.append
237
238 for tok in chain([token], iterable):
239 toknum, tokval = tok[:2]
240 if toknum == ENCODING:
241 self.encoding = tokval
242 continue
243
244 if toknum in (NAME, NUMBER):
245 tokval += ' '
246
247 # Insert a space between two consecutive strings
248 if toknum == STRING:
249 if prevstring:
250 tokval = ' ' + tokval
251 prevstring = True
252 else:
253 prevstring = False
254
255 if toknum == INDENT:
256 indents.append(tokval)
257 continue
258 elif toknum == DEDENT:
259 indents.pop()
260 continue
261 elif toknum in (NEWLINE, NL):
262 startline = True
263 elif startline and indents:
264 toks_append(indents[-1])
265 startline = False
266 toks_append(tokval)
267
268
269 def untokenize(tokens):
270 """
271 Convert ``tokens`` (an iterable) back into Python source code. Return
272 a bytes object, encoded using the encoding specified by the last
273 ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
274
275 The result is guaranteed to tokenize back to match the input so that
276 the conversion is lossless and round-trips are assured. The
277 guarantee applies only to the token type and token string as the
278 spacing between tokens (column positions) may change.
279
280 :func:`untokenize` has two modes. If the input tokens are sequences
281 of length 2 (``type``, ``string``) then spaces are added as necessary to
282 preserve the round-trip property.
283
284 If the input tokens are sequences of length 4 or more (``type``,
285 ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
286 spaces are added so that each token appears in the result at the
287 position indicated by ``start`` and ``end``, if possible.
288 """
289 return Untokenizer().untokenize(tokens)
290
291
292 def _get_normal_name(orig_enc):
293 """Imitates get_normal_name in tokenizer.c."""
294 # Only care about the first 12 characters.
295 enc = orig_enc[:12].lower().replace("_", "-")
296 if enc == "utf-8" or enc.startswith("utf-8-"):
297 return "utf-8"
298 if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
299 enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
300 return "iso-8859-1"
301 return orig_enc
302
303 def detect_encoding(readline):
304 """
305 The detect_encoding() function is used to detect the encoding that should
306 be used to decode a Python source file. It requires one argment, readline,
307 in the same way as the tokenize() generator.
308
309 It will call readline a maximum of twice, and return the encoding used
310 (as a string) and a list of any lines (left as bytes) it has read in.
311
312 It detects the encoding from the presence of a utf-8 bom or an encoding
313 cookie as specified in pep-0263. If both a bom and a cookie are present,
314 but disagree, a SyntaxError will be raised. If the encoding cookie is an
315 invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
316 'utf-8-sig' is returned.
317
318 If no encoding is specified, then the default of 'utf-8' will be returned.
319 """
320 bom_found = False
321 encoding = None
322 default = 'utf-8'
323 def read_or_stop():
324 try:
325 return readline()
326 except StopIteration:
327 return b''
328
329 def find_cookie(line):
330 try:
331 # Decode as UTF-8. Either the line is an encoding declaration,
332 # in which case it should be pure ASCII, or it must be UTF-8
333 # per default encoding.
334 line_string = line.decode('utf-8')
335 except UnicodeDecodeError:
336 raise SyntaxError("invalid or missing encoding declaration")
337
338 matches = cookie_re.findall(line_string)
339 if not matches:
340 return None
341 encoding = _get_normal_name(matches[0])
342 try:
343 codec = lookup(encoding)
344 except LookupError:
345 # This behaviour mimics the Python interpreter
346 raise SyntaxError("unknown encoding: " + encoding)
347
348 if bom_found:
349 if encoding != 'utf-8':
350 # This behaviour mimics the Python interpreter
351 raise SyntaxError('encoding problem: utf-8')
352 encoding += '-sig'
353 return encoding
354
355 first = read_or_stop()
356 if first.startswith(BOM_UTF8):
357 bom_found = True
358 first = first[3:]
359 default = 'utf-8-sig'
360 if not first:
361 return default, []
362
363 encoding = find_cookie(first)
364 if encoding:
365 return encoding, [first]
366
367 second = read_or_stop()
368 if not second:
369 return default, [first]
370
371 encoding = find_cookie(second)
372 if encoding:
373 return encoding, [first, second]
374
375 return default, [first, second]
376
377
378 def open(filename):
379 """Open a file in read only mode using the encoding detected by
380 detect_encoding().
381 """
382 buffer = builtins.open(filename, 'rb')
383 encoding, lines = detect_encoding(buffer.readline)
384 buffer.seek(0)
385 text = TextIOWrapper(buffer, encoding, line_buffering=True)
386 text.mode = 'r'
387 return text
388
389
390 def tokenize(readline):
391 """
392 The tokenize() generator requires one argment, readline, which
393 must be a callable object which provides the same interface as the
394 readline() method of built-in file objects. Each call to the function
395 should return one line of input as bytes. Alternately, readline
396 can be a callable function terminating with StopIteration:
397 readline = open(myfile, 'rb').__next__ # Example of alternate readline
398
399 The generator produces 5-tuples with these members: the token type; the
400 token string; a 2-tuple (srow, scol) of ints specifying the row and
401 column where the token begins in the source; a 2-tuple (erow, ecol) of
402 ints specifying the row and column where the token ends in the source;
403 and the line on which the token was found. The line passed is the
404 logical line; continuation lines are included.
405
406 The first token sequence will always be an ENCODING token
407 which tells you which encoding was used to decode the bytes stream.
408 """
409 # This import is here to avoid problems when the itertools module is not
410 # built yet and tokenize is imported.
411 from itertools import chain, repeat
412 encoding, consumed = detect_encoding(readline)
413 rl_gen = iter(readline, b"")
414 empty = repeat(b"")
415 return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
416
417
418 def _tokenize(readline, encoding):
419 lnum = parenlev = continued = 0
420 numchars = '0123456789'
421 contstr, needcont = '', 0
422 contline = None
423 indents = [0]
424
425 if encoding is not None:
426 if encoding == "utf-8-sig":
427 # BOM will already have been stripped.
428 encoding = "utf-8"
429 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
430 while True: # loop over lines in stream
431 try:
432 line = readline()
433 except StopIteration:
434 line = b''
435
436 if encoding is not None:
437 line = line.decode(encoding)
438 lnum += 1
439 pos, max = 0, len(line)
440
441 if contstr: # continued string
442 if not line:
443 raise TokenError("EOF in multi-line string", strstart)
444 endmatch = endprog.match(line)
445 if endmatch:
446 pos = end = endmatch.end(0)
447 yield TokenInfo(STRING, contstr + line[:end],
448 strstart, (lnum, end), contline + line)
449 contstr, needcont = '', 0
450 contline = None
451 elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
452 yield TokenInfo(ERRORTOKEN, contstr + line,
453 strstart, (lnum, len(line)), contline)
454 contstr = ''
455 contline = None
456 continue
457 else:
458 contstr = contstr + line
459 contline = contline + line
460 continue
461
462 elif parenlev == 0 and not continued: # new statement
463 if not line: break
464 column = 0
465 while pos < max: # measure leading whitespace
466 if line[pos] == ' ':
467 column += 1
468 elif line[pos] == '\t':
469 column = (column//tabsize + 1)*tabsize
470 elif line[pos] == '\f':
471 column = 0
472 else:
473 break
474 pos += 1
475 if pos == max:
476 break
477
478 if line[pos] in '#\r\n': # skip comments or blank lines
479 if line[pos] == '#':
480 comment_token = line[pos:].rstrip('\r\n')
481 nl_pos = pos + len(comment_token)
482 yield TokenInfo(COMMENT, comment_token,
483 (lnum, pos), (lnum, pos + len(comment_token)), line)
484 yield TokenInfo(NEWLINE, line[nl_pos:],
485 (lnum, nl_pos), (lnum, len(line)), line)
486 else:
487 yield TokenInfo(NEWLINE, line[pos:],
488 (lnum, pos), (lnum, len(line)), line)
489 continue
490
491 if column > indents[-1]: # count indents or dedents
492 indents.append(column)
493 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
494 while column < indents[-1]:
495 if column not in indents:
496 raise IndentationError(
497 "unindent does not match any outer indentation level",
498 ("<tokenize>", lnum, pos, line))
499 indents = indents[:-1]
500 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
501
502 else: # continued statement
503 if not line:
504 raise TokenError("EOF in multi-line statement", (lnum, 0))
505 continued = 0
506
507 while pos < max:
508 pseudomatch = pseudoprog.match(line, pos)
509 if pseudomatch: # scan for tokens
510 start, end = pseudomatch.span(1)
511 spos, epos, pos = (lnum, start), (lnum, end), end
512 token, initial = line[start:end], line[start]
513
514 if (initial in numchars or # ordinary number
515 (initial == '.' and token != '.' and token != '...')):
516 yield TokenInfo(NUMBER, token, spos, epos, line)
517 elif initial in '\r\n':
518 yield TokenInfo(NL if parenlev > 0 else NEWLINE,
519 token, spos, epos, line)
520 elif initial == '#':
521 assert not token.endswith("\n")
522 yield TokenInfo(COMMENT, token, spos, epos, line)
523 elif token in triple_quoted:
524 endprog = endprogs[token]
525 endmatch = endprog.match(line, pos)
526 if endmatch: # all on one line
527 pos = endmatch.end(0)
528 token = line[start:pos]
529 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
530 else:
531 strstart = (lnum, start) # multiple lines
532 contstr = line[start:]
533 contline = line
534 break
535 elif initial in single_quoted or \
536 token[:2] in single_quoted or \
537 token[:3] in single_quoted:
538 if token[-1] == '\n': # continued string
539 strstart = (lnum, start)
540 endprog = (endprogs[initial] or endprogs[token[1]] or
541 endprogs[token[2]])
542 contstr, needcont = line[start:], 1
543 contline = line
544 break
545 else: # ordinary string
546 yield TokenInfo(STRING, token, spos, epos, line)
547 elif initial.isidentifier(): # ordinary name
548 yield TokenInfo(NAME, token, spos, epos, line)
549 elif initial == '\\': # continued stmt
550 continued = 1
551 else:
552 if initial in '([{':
553 parenlev += 1
554 elif initial in ')]}':
555 parenlev -= 1
556 yield TokenInfo(OP, token, spos, epos, line)
557 else:
558 yield TokenInfo(ERRORTOKEN, line[pos],
559 (lnum, pos), (lnum, pos+1), line)
560 pos += 1
561
562 for indent in indents[1:]: # pop remaining indent levels
563 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
564 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
565
566
567 # An undocumented, backwards compatible, API for all the places in the standard
568 # library that expect to be able to use tokenize with strings
569 def generate_tokens(readline):
570 return _tokenize(readline, None)
571
572 if __name__ == "__main__":
573 # Quick sanity check
574 s = b'''def parseline(self, line):
575 """Parse the line into a command name and a string containing
576 the arguments. Returns a tuple containing (command, args, line).
577 'command' and 'args' may be None if the line couldn't be parsed.
578 """
579 line = line.strip()
580 if not line:
581 return None, None, line
582 elif line[0] == '?':
583 line = 'help ' + line[1:]
584 elif line[0] == '!':
585 if hasattr(self, 'do_shell'):
586 line = 'shell ' + line[1:]
587 else:
588 return None, None, line
589 i, n = 0, len(line)
590 while i < n and line[i] in self.identchars: i = i+1
591 cmd, arg = line[:i], line[i:].strip()
592 return cmd, arg, line
593 '''
594 for tok in tokenize(iter(s.splitlines()).__next__):
595 print(tok)
General Comments 0
You need to be logged in to leave comments. Login now