##// END OF EJS Templates
Fix ResourceWarning: unclosed file...
Mickaël Schoentgen -
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -1,225 +1,226 b''
1 1 # encoding: utf-8
2 2 """sys.excepthook for IPython itself, leaves a detailed report on disk.
3 3
4 4 Authors:
5 5
6 6 * Fernando Perez
7 7 * Brian E. Granger
8 8 """
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
12 12 # Copyright (C) 2008-2011 The IPython Development Team
13 13 #
14 14 # Distributed under the terms of the BSD License. The full license is in
15 15 # the file COPYING, distributed as part of this software.
16 16 #-----------------------------------------------------------------------------
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Imports
20 20 #-----------------------------------------------------------------------------
21 21
22 22 import os
23 23 import sys
24 24 import traceback
25 25 from pprint import pformat
26 26
27 27 from IPython.core import ultratb
28 28 from IPython.core.release import author_email
29 29 from IPython.utils.sysinfo import sys_info
30 30 from IPython.utils.py3compat import input
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Code
34 34 #-----------------------------------------------------------------------------
35 35
36 36 # Template for the user message.
37 37 _default_message_template = """\
38 38 Oops, {app_name} crashed. We do our best to make it stable, but...
39 39
40 40 A crash report was automatically generated with the following information:
41 41 - A verbatim copy of the crash traceback.
42 42 - A copy of your input history during this session.
43 43 - Data on your current {app_name} configuration.
44 44
45 45 It was left in the file named:
46 46 \t'{crash_report_fname}'
47 47 If you can email this file to the developers, the information in it will help
48 48 them in understanding and correcting the problem.
49 49
50 50 You can mail it to: {contact_name} at {contact_email}
51 51 with the subject '{app_name} Crash Report'.
52 52
53 53 If you want to do it now, the following command will work (under Unix):
54 54 mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
55 55
56 56 In your email, please also include information about:
57 57 - The operating system under which the crash happened: Linux, macOS, Windows,
58 58 other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
59 59 Windows 10 Pro), and whether it is 32-bit or 64-bit;
60 60 - How {app_name} was installed: using pip or conda, from GitHub, as part of
61 61 a Docker container, or other, providing more detail if possible;
62 62 - How to reproduce the crash: what exact sequence of instructions can one
63 63 input to get the same crash? Ideally, find a minimal yet complete sequence
64 64 of instructions that yields the crash.
65 65
66 66 To ensure accurate tracking of this issue, please file a report about it at:
67 67 {bug_tracker}
68 68 """
69 69
70 70 _lite_message_template = """
71 71 If you suspect this is an IPython bug, please report it at:
72 72 https://github.com/ipython/ipython/issues
73 73 or send an email to the mailing list at {email}
74 74
75 75 You can print a more detailed traceback right now with "%tb", or use "%debug"
76 76 to interactively debug it.
77 77
78 78 Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
79 79 {config}Application.verbose_crash=True
80 80 """
81 81
82 82
83 83 class CrashHandler(object):
84 84 """Customizable crash handlers for IPython applications.
85 85
86 86 Instances of this class provide a :meth:`__call__` method which can be
87 87 used as a ``sys.excepthook``. The :meth:`__call__` signature is::
88 88
89 89 def __call__(self, etype, evalue, etb)
90 90 """
91 91
92 92 message_template = _default_message_template
93 93 section_sep = '\n\n'+'*'*75+'\n\n'
94 94
95 95 def __init__(self, app, contact_name=None, contact_email=None,
96 96 bug_tracker=None, show_crash_traceback=True, call_pdb=False):
97 97 """Create a new crash handler
98 98
99 99 Parameters
100 100 ----------
101 101 app : Application
102 102 A running :class:`Application` instance, which will be queried at
103 103 crash time for internal information.
104 104
105 105 contact_name : str
106 106 A string with the name of the person to contact.
107 107
108 108 contact_email : str
109 109 A string with the email address of the contact.
110 110
111 111 bug_tracker : str
112 112 A string with the URL for your project's bug tracker.
113 113
114 114 show_crash_traceback : bool
115 115 If false, don't print the crash traceback on stderr, only generate
116 116 the on-disk report
117 117
118 118 Non-argument instance attributes:
119 119
120 120 These instances contain some non-argument attributes which allow for
121 121 further customization of the crash handler's behavior. Please see the
122 122 source for further details.
123 123 """
124 124 self.crash_report_fname = "Crash_report_%s.txt" % app.name
125 125 self.app = app
126 126 self.call_pdb = call_pdb
127 127 #self.call_pdb = True # dbg
128 128 self.show_crash_traceback = show_crash_traceback
129 129 self.info = dict(app_name = app.name,
130 130 contact_name = contact_name,
131 131 contact_email = contact_email,
132 132 bug_tracker = bug_tracker,
133 133 crash_report_fname = self.crash_report_fname)
134 134
135 135
136 136 def __call__(self, etype, evalue, etb):
137 137 """Handle an exception, call for compatible with sys.excepthook"""
138 138
139 139 # do not allow the crash handler to be called twice without reinstalling it
140 140 # this prevents unlikely errors in the crash handling from entering an
141 141 # infinite loop.
142 142 sys.excepthook = sys.__excepthook__
143 143
144 144 # Report tracebacks shouldn't use color in general (safer for users)
145 145 color_scheme = 'NoColor'
146 146
147 147 # Use this ONLY for developer debugging (keep commented out for release)
148 148 #color_scheme = 'Linux' # dbg
149 149 try:
150 150 rptdir = self.app.ipython_dir
151 151 except:
152 152 rptdir = os.getcwd()
153 153 if rptdir is None or not os.path.isdir(rptdir):
154 154 rptdir = os.getcwd()
155 155 report_name = os.path.join(rptdir,self.crash_report_fname)
156 156 # write the report filename into the instance dict so it can get
157 157 # properly expanded out in the user message template
158 158 self.crash_report_fname = report_name
159 159 self.info['crash_report_fname'] = report_name
160 160 TBhandler = ultratb.VerboseTB(
161 161 color_scheme=color_scheme,
162 162 long_header=1,
163 163 call_pdb=self.call_pdb,
164 164 )
165 165 if self.call_pdb:
166 166 TBhandler(etype,evalue,etb)
167 167 return
168 168 else:
169 169 traceback = TBhandler.text(etype,evalue,etb,context=31)
170 170
171 171 # print traceback to screen
172 172 if self.show_crash_traceback:
173 173 print(traceback, file=sys.stderr)
174 174
175 175 # and generate a complete report on disk
176 176 try:
177 177 report = open(report_name,'w')
178 178 except:
179 179 print('Could not create crash report on disk.', file=sys.stderr)
180 180 return
181 181
182 # Inform user on stderr of what happened
183 print('\n'+'*'*70+'\n', file=sys.stderr)
184 print(self.message_template.format(**self.info), file=sys.stderr)
182 with report:
183 # Inform user on stderr of what happened
184 print('\n'+'*'*70+'\n', file=sys.stderr)
185 print(self.message_template.format(**self.info), file=sys.stderr)
186
187 # Construct report on disk
188 report.write(self.make_report(traceback))
185 189
186 # Construct report on disk
187 report.write(self.make_report(traceback))
188 report.close()
189 190 input("Hit <Enter> to quit (your terminal may close):")
190 191
191 192 def make_report(self,traceback):
192 193 """Return a string containing a crash report."""
193 194
194 195 sec_sep = self.section_sep
195 196
196 197 report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
197 198 rpt_add = report.append
198 199 rpt_add(sys_info())
199 200
200 201 try:
201 202 config = pformat(self.app.config)
202 203 rpt_add(sec_sep)
203 204 rpt_add('Application name: %s\n\n' % self.app_name)
204 205 rpt_add('Current user configuration structure:\n\n')
205 206 rpt_add(config)
206 207 except:
207 208 pass
208 209 rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
209 210
210 211 return ''.join(report)
211 212
212 213
213 214 def crash_handler_lite(etype, evalue, tb):
214 215 """a light excepthook, adding a small message to the usual traceback"""
215 216 traceback.print_exception(etype, evalue, tb)
216 217
217 218 from IPython.core.interactiveshell import InteractiveShell
218 219 if InteractiveShell.initialized():
219 220 # we are in a Shell environment, give %magic example
220 221 config = "%config "
221 222 else:
222 223 # we are not in a shell, show generic config
223 224 config = "c."
224 225 print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
225 226
@@ -1,645 +1,644 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 Pdb debugger class.
4 4
5 5 Modified from the standard pdb.Pdb class to avoid including readline, so that
6 6 the command line completion of other programs which include this isn't
7 7 damaged.
8 8
9 9 In the future, this class will be expanded with improvements over the standard
10 10 pdb.
11 11
12 12 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
13 13 changes. Licensing should therefore be under the standard Python terms. For
14 14 details on the PSF (Python Software Foundation) standard license, see:
15 15
16 16 https://docs.python.org/2/license.html
17 17 """
18 18
19 19 #*****************************************************************************
20 20 #
21 21 # This file is licensed under the PSF license.
22 22 #
23 23 # Copyright (C) 2001 Python Software Foundation, www.python.org
24 24 # Copyright (C) 2005-2006 Fernando Perez. <fperez@colorado.edu>
25 25 #
26 26 #
27 27 #*****************************************************************************
28 28
29 29 import bdb
30 30 import functools
31 31 import inspect
32 32 import linecache
33 33 import sys
34 34 import warnings
35 35 import re
36 36
37 37 from IPython import get_ipython
38 38 from IPython.utils import PyColorize
39 39 from IPython.utils import coloransi, py3compat
40 40 from IPython.core.excolors import exception_colors
41 41 from IPython.testing.skipdoctest import skip_doctest
42 42
43 43
44 44 prompt = 'ipdb> '
45 45
46 46 #We have to check this directly from sys.argv, config struct not yet available
47 47 from pdb import Pdb as OldPdb
48 48
49 49 # Allow the set_trace code to operate outside of an ipython instance, even if
50 50 # it does so with some limitations. The rest of this support is implemented in
51 51 # the Tracer constructor.
52 52
53 53 def make_arrow(pad):
54 54 """generate the leading arrow in front of traceback or debugger"""
55 55 if pad >= 2:
56 56 return '-'*(pad-2) + '> '
57 57 elif pad == 1:
58 58 return '>'
59 59 return ''
60 60
61 61
62 62 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
63 63 """Exception hook which handles `BdbQuit` exceptions.
64 64
65 65 All other exceptions are processed using the `excepthook`
66 66 parameter.
67 67 """
68 68 warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
69 69 DeprecationWarning, stacklevel=2)
70 70 if et==bdb.BdbQuit:
71 71 print('Exiting Debugger.')
72 72 elif excepthook is not None:
73 73 excepthook(et, ev, tb)
74 74 else:
75 75 # Backwards compatibility. Raise deprecation warning?
76 76 BdbQuit_excepthook.excepthook_ori(et,ev,tb)
77 77
78 78
79 79 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
80 80 warnings.warn(
81 81 "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
82 82 DeprecationWarning, stacklevel=2)
83 83 print('Exiting Debugger.')
84 84
85 85
86 86 class Tracer(object):
87 87 """
88 88 DEPRECATED
89 89
90 90 Class for local debugging, similar to pdb.set_trace.
91 91
92 92 Instances of this class, when called, behave like pdb.set_trace, but
93 93 providing IPython's enhanced capabilities.
94 94
95 95 This is implemented as a class which must be initialized in your own code
96 96 and not as a standalone function because we need to detect at runtime
97 97 whether IPython is already active or not. That detection is done in the
98 98 constructor, ensuring that this code plays nicely with a running IPython,
99 99 while functioning acceptably (though with limitations) if outside of it.
100 100 """
101 101
102 102 @skip_doctest
103 103 def __init__(self, colors=None):
104 104 """
105 105 DEPRECATED
106 106
107 107 Create a local debugger instance.
108 108
109 109 Parameters
110 110 ----------
111 111
112 112 colors : str, optional
113 113 The name of the color scheme to use, it must be one of IPython's
114 114 valid color schemes. If not given, the function will default to
115 115 the current IPython scheme when running inside IPython, and to
116 116 'NoColor' otherwise.
117 117
118 118 Examples
119 119 --------
120 120 ::
121 121
122 122 from IPython.core.debugger import Tracer; debug_here = Tracer()
123 123
124 124 Later in your code::
125 125
126 126 debug_here() # -> will open up the debugger at that point.
127 127
128 128 Once the debugger activates, you can use all of its regular commands to
129 129 step through code, set breakpoints, etc. See the pdb documentation
130 130 from the Python standard library for usage details.
131 131 """
132 132 warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
133 133 "`IPython.core.debugger.Pdb.set_trace()`",
134 134 DeprecationWarning, stacklevel=2)
135 135
136 136 ip = get_ipython()
137 137 if ip is None:
138 138 # Outside of ipython, we set our own exception hook manually
139 139 sys.excepthook = functools.partial(BdbQuit_excepthook,
140 140 excepthook=sys.excepthook)
141 141 def_colors = 'NoColor'
142 142 else:
143 143 # In ipython, we use its custom exception handler mechanism
144 144 def_colors = ip.colors
145 145 ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
146 146
147 147 if colors is None:
148 148 colors = def_colors
149 149
150 150 # The stdlib debugger internally uses a modified repr from the `repr`
151 151 # module, that limits the length of printed strings to a hardcoded
152 152 # limit of 30 characters. That much trimming is too aggressive, let's
153 153 # at least raise that limit to 80 chars, which should be enough for
154 154 # most interactive uses.
155 155 try:
156 156 try:
157 157 from reprlib import aRepr # Py 3
158 158 except ImportError:
159 159 from repr import aRepr # Py 2
160 160 aRepr.maxstring = 80
161 161 except:
162 162 # This is only a user-facing convenience, so any error we encounter
163 163 # here can be warned about but can be otherwise ignored. These
164 164 # printouts will tell us about problems if this API changes
165 165 import traceback
166 166 traceback.print_exc()
167 167
168 168 self.debugger = Pdb(colors)
169 169
170 170 def __call__(self):
171 171 """Starts an interactive debugger at the point where called.
172 172
173 173 This is similar to the pdb.set_trace() function from the std lib, but
174 174 using IPython's enhanced debugger."""
175 175
176 176 self.debugger.set_trace(sys._getframe().f_back)
177 177
178 178
179 179 RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
180 180
181 181
182 182 def strip_indentation(multiline_string):
183 183 return RGX_EXTRA_INDENT.sub('', multiline_string)
184 184
185 185
186 186 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
187 187 """Make new_fn have old_fn's doc string. This is particularly useful
188 188 for the ``do_...`` commands that hook into the help system.
189 189 Adapted from from a comp.lang.python posting
190 190 by Duncan Booth."""
191 191 def wrapper(*args, **kw):
192 192 return new_fn(*args, **kw)
193 193 if old_fn.__doc__:
194 194 wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
195 195 return wrapper
196 196
197 197
198 198 def _file_lines(fname):
199 199 """Return the contents of a named file as a list of lines.
200 200
201 201 This function never raises an IOError exception: if the file can't be
202 202 read, it simply returns an empty list."""
203 203
204 204 try:
205 205 outfile = open(fname)
206 206 except IOError:
207 207 return []
208 208 else:
209 out = outfile.readlines()
210 outfile.close()
211 return out
209 with out:
210 return outfile.readlines()
212 211
213 212
214 213 class Pdb(OldPdb):
215 214 """Modified Pdb class, does not load readline.
216 215
217 216 for a standalone version that uses prompt_toolkit, see
218 217 `IPython.terminal.debugger.TerminalPdb` and
219 218 `IPython.terminal.debugger.set_trace()`
220 219 """
221 220
222 221 def __init__(self, color_scheme=None, completekey=None,
223 222 stdin=None, stdout=None, context=5):
224 223
225 224 # Parent constructor:
226 225 try:
227 226 self.context = int(context)
228 227 if self.context <= 0:
229 228 raise ValueError("Context must be a positive integer")
230 229 except (TypeError, ValueError):
231 230 raise ValueError("Context must be a positive integer")
232 231
233 232 OldPdb.__init__(self, completekey, stdin, stdout)
234 233
235 234 # IPython changes...
236 235 self.shell = get_ipython()
237 236
238 237 if self.shell is None:
239 238 save_main = sys.modules['__main__']
240 239 # No IPython instance running, we must create one
241 240 from IPython.terminal.interactiveshell import \
242 241 TerminalInteractiveShell
243 242 self.shell = TerminalInteractiveShell.instance()
244 243 # needed by any code which calls __import__("__main__") after
245 244 # the debugger was entered. See also #9941.
246 245 sys.modules['__main__'] = save_main
247 246
248 247 if color_scheme is not None:
249 248 warnings.warn(
250 249 "The `color_scheme` argument is deprecated since version 5.1",
251 250 DeprecationWarning, stacklevel=2)
252 251 else:
253 252 color_scheme = self.shell.colors
254 253
255 254 self.aliases = {}
256 255
257 256 # Create color table: we copy the default one from the traceback
258 257 # module and add a few attributes needed for debugging
259 258 self.color_scheme_table = exception_colors()
260 259
261 260 # shorthands
262 261 C = coloransi.TermColors
263 262 cst = self.color_scheme_table
264 263
265 264 cst['NoColor'].colors.prompt = C.NoColor
266 265 cst['NoColor'].colors.breakpoint_enabled = C.NoColor
267 266 cst['NoColor'].colors.breakpoint_disabled = C.NoColor
268 267
269 268 cst['Linux'].colors.prompt = C.Green
270 269 cst['Linux'].colors.breakpoint_enabled = C.LightRed
271 270 cst['Linux'].colors.breakpoint_disabled = C.Red
272 271
273 272 cst['LightBG'].colors.prompt = C.Blue
274 273 cst['LightBG'].colors.breakpoint_enabled = C.LightRed
275 274 cst['LightBG'].colors.breakpoint_disabled = C.Red
276 275
277 276 cst['Neutral'].colors.prompt = C.Blue
278 277 cst['Neutral'].colors.breakpoint_enabled = C.LightRed
279 278 cst['Neutral'].colors.breakpoint_disabled = C.Red
280 279
281 280
282 281 # Add a python parser so we can syntax highlight source while
283 282 # debugging.
284 283 self.parser = PyColorize.Parser(style=color_scheme)
285 284 self.set_colors(color_scheme)
286 285
287 286 # Set the prompt - the default prompt is '(Pdb)'
288 287 self.prompt = prompt
289 288
290 289 def set_colors(self, scheme):
291 290 """Shorthand access to the color table scheme selector method."""
292 291 self.color_scheme_table.set_active_scheme(scheme)
293 292 self.parser.style = scheme
294 293
295 294 def interaction(self, frame, traceback):
296 295 try:
297 296 OldPdb.interaction(self, frame, traceback)
298 297 except KeyboardInterrupt:
299 298 sys.stdout.write('\n' + self.shell.get_exception_only())
300 299
301 300 def new_do_up(self, arg):
302 301 OldPdb.do_up(self, arg)
303 302 do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
304 303
305 304 def new_do_down(self, arg):
306 305 OldPdb.do_down(self, arg)
307 306
308 307 do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
309 308
310 309 def new_do_frame(self, arg):
311 310 OldPdb.do_frame(self, arg)
312 311
313 312 def new_do_quit(self, arg):
314 313
315 314 if hasattr(self, 'old_all_completions'):
316 315 self.shell.Completer.all_completions=self.old_all_completions
317 316
318 317 return OldPdb.do_quit(self, arg)
319 318
320 319 do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
321 320
322 321 def new_do_restart(self, arg):
323 322 """Restart command. In the context of ipython this is exactly the same
324 323 thing as 'quit'."""
325 324 self.msg("Restart doesn't make sense here. Using 'quit' instead.")
326 325 return self.do_quit(arg)
327 326
328 327 def print_stack_trace(self, context=None):
329 328 if context is None:
330 329 context = self.context
331 330 try:
332 331 context=int(context)
333 332 if context <= 0:
334 333 raise ValueError("Context must be a positive integer")
335 334 except (TypeError, ValueError):
336 335 raise ValueError("Context must be a positive integer")
337 336 try:
338 337 for frame_lineno in self.stack:
339 338 self.print_stack_entry(frame_lineno, context=context)
340 339 except KeyboardInterrupt:
341 340 pass
342 341
343 342 def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
344 343 context=None):
345 344 if context is None:
346 345 context = self.context
347 346 try:
348 347 context=int(context)
349 348 if context <= 0:
350 349 raise ValueError("Context must be a positive integer")
351 350 except (TypeError, ValueError):
352 351 raise ValueError("Context must be a positive integer")
353 352 print(self.format_stack_entry(frame_lineno, '', context))
354 353
355 354 # vds: >>
356 355 frame, lineno = frame_lineno
357 356 filename = frame.f_code.co_filename
358 357 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
359 358 # vds: <<
360 359
361 360 def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
362 361 if context is None:
363 362 context = self.context
364 363 try:
365 364 context=int(context)
366 365 if context <= 0:
367 366 print("Context must be a positive integer")
368 367 except (TypeError, ValueError):
369 368 print("Context must be a positive integer")
370 369 try:
371 370 import reprlib # Py 3
372 371 except ImportError:
373 372 import repr as reprlib # Py 2
374 373
375 374 ret = []
376 375
377 376 Colors = self.color_scheme_table.active_colors
378 377 ColorsNormal = Colors.Normal
379 378 tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
380 379 tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
381 380 tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
382 381 tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
383 382 ColorsNormal)
384 383
385 384 frame, lineno = frame_lineno
386 385
387 386 return_value = ''
388 387 if '__return__' in frame.f_locals:
389 388 rv = frame.f_locals['__return__']
390 389 #return_value += '->'
391 390 return_value += reprlib.repr(rv) + '\n'
392 391 ret.append(return_value)
393 392
394 393 #s = filename + '(' + `lineno` + ')'
395 394 filename = self.canonic(frame.f_code.co_filename)
396 395 link = tpl_link % py3compat.cast_unicode(filename)
397 396
398 397 if frame.f_code.co_name:
399 398 func = frame.f_code.co_name
400 399 else:
401 400 func = "<lambda>"
402 401
403 402 call = ''
404 403 if func != '?':
405 404 if '__args__' in frame.f_locals:
406 405 args = reprlib.repr(frame.f_locals['__args__'])
407 406 else:
408 407 args = '()'
409 408 call = tpl_call % (func, args)
410 409
411 410 # The level info should be generated in the same format pdb uses, to
412 411 # avoid breaking the pdbtrack functionality of python-mode in *emacs.
413 412 if frame is self.curframe:
414 413 ret.append('> ')
415 414 else:
416 415 ret.append(' ')
417 416 ret.append(u'%s(%s)%s\n' % (link,lineno,call))
418 417
419 418 start = lineno - 1 - context//2
420 419 lines = linecache.getlines(filename)
421 420 start = min(start, len(lines) - context)
422 421 start = max(start, 0)
423 422 lines = lines[start : start + context]
424 423
425 424 for i,line in enumerate(lines):
426 425 show_arrow = (start + 1 + i == lineno)
427 426 linetpl = (frame is self.curframe or show_arrow) \
428 427 and tpl_line_em \
429 428 or tpl_line
430 429 ret.append(self.__format_line(linetpl, filename,
431 430 start + 1 + i, line,
432 431 arrow = show_arrow) )
433 432 return ''.join(ret)
434 433
435 434 def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
436 435 bp_mark = ""
437 436 bp_mark_color = ""
438 437
439 438 new_line, err = self.parser.format2(line, 'str')
440 439 if not err:
441 440 line = new_line
442 441
443 442 bp = None
444 443 if lineno in self.get_file_breaks(filename):
445 444 bps = self.get_breaks(filename, lineno)
446 445 bp = bps[-1]
447 446
448 447 if bp:
449 448 Colors = self.color_scheme_table.active_colors
450 449 bp_mark = str(bp.number)
451 450 bp_mark_color = Colors.breakpoint_enabled
452 451 if not bp.enabled:
453 452 bp_mark_color = Colors.breakpoint_disabled
454 453
455 454 numbers_width = 7
456 455 if arrow:
457 456 # This is the line with the error
458 457 pad = numbers_width - len(str(lineno)) - len(bp_mark)
459 458 num = '%s%s' % (make_arrow(pad), str(lineno))
460 459 else:
461 460 num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
462 461
463 462 return tpl_line % (bp_mark_color + bp_mark, num, line)
464 463
465 464
466 465 def print_list_lines(self, filename, first, last):
467 466 """The printing (as opposed to the parsing part of a 'list'
468 467 command."""
469 468 try:
470 469 Colors = self.color_scheme_table.active_colors
471 470 ColorsNormal = Colors.Normal
472 471 tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
473 472 tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
474 473 src = []
475 474 if filename == "<string>" and hasattr(self, "_exec_filename"):
476 475 filename = self._exec_filename
477 476
478 477 for lineno in range(first, last+1):
479 478 line = linecache.getline(filename, lineno)
480 479 if not line:
481 480 break
482 481
483 482 if lineno == self.curframe.f_lineno:
484 483 line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
485 484 else:
486 485 line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
487 486
488 487 src.append(line)
489 488 self.lineno = lineno
490 489
491 490 print(''.join(src))
492 491
493 492 except KeyboardInterrupt:
494 493 pass
495 494
496 495 def do_list(self, arg):
497 496 """Print lines of code from the current stack frame
498 497 """
499 498 self.lastcmd = 'list'
500 499 last = None
501 500 if arg:
502 501 try:
503 502 x = eval(arg, {}, {})
504 503 if type(x) == type(()):
505 504 first, last = x
506 505 first = int(first)
507 506 last = int(last)
508 507 if last < first:
509 508 # Assume it's a count
510 509 last = first + last
511 510 else:
512 511 first = max(1, int(x) - 5)
513 512 except:
514 513 print('*** Error in argument:', repr(arg))
515 514 return
516 515 elif self.lineno is None:
517 516 first = max(1, self.curframe.f_lineno - 5)
518 517 else:
519 518 first = self.lineno + 1
520 519 if last is None:
521 520 last = first + 10
522 521 self.print_list_lines(self.curframe.f_code.co_filename, first, last)
523 522
524 523 # vds: >>
525 524 lineno = first
526 525 filename = self.curframe.f_code.co_filename
527 526 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
528 527 # vds: <<
529 528
530 529 do_l = do_list
531 530
532 531 def getsourcelines(self, obj):
533 532 lines, lineno = inspect.findsource(obj)
534 533 if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
535 534 # must be a module frame: do not try to cut a block out of it
536 535 return lines, 1
537 536 elif inspect.ismodule(obj):
538 537 return lines, 1
539 538 return inspect.getblock(lines[lineno:]), lineno+1
540 539
541 540 def do_longlist(self, arg):
542 541 """Print lines of code from the current stack frame.
543 542
544 543 Shows more lines than 'list' does.
545 544 """
546 545 self.lastcmd = 'longlist'
547 546 try:
548 547 lines, lineno = self.getsourcelines(self.curframe)
549 548 except OSError as err:
550 549 self.error(err)
551 550 return
552 551 last = lineno + len(lines)
553 552 self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
554 553 do_ll = do_longlist
555 554
556 555 def do_debug(self, arg):
557 556 """debug code
558 557 Enter a recursive debugger that steps through the code
559 558 argument (which is an arbitrary expression or statement to be
560 559 executed in the current environment).
561 560 """
562 561 sys.settrace(None)
563 562 globals = self.curframe.f_globals
564 563 locals = self.curframe_locals
565 564 p = self.__class__(completekey=self.completekey,
566 565 stdin=self.stdin, stdout=self.stdout)
567 566 p.use_rawinput = self.use_rawinput
568 567 p.prompt = "(%s) " % self.prompt.strip()
569 568 self.message("ENTERING RECURSIVE DEBUGGER")
570 569 sys.call_tracing(p.run, (arg, globals, locals))
571 570 self.message("LEAVING RECURSIVE DEBUGGER")
572 571 sys.settrace(self.trace_dispatch)
573 572 self.lastcmd = p.lastcmd
574 573
575 574 def do_pdef(self, arg):
576 575 """Print the call signature for any callable object.
577 576
578 577 The debugger interface to %pdef"""
579 578 namespaces = [('Locals', self.curframe.f_locals),
580 579 ('Globals', self.curframe.f_globals)]
581 580 self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
582 581
583 582 def do_pdoc(self, arg):
584 583 """Print the docstring for an object.
585 584
586 585 The debugger interface to %pdoc."""
587 586 namespaces = [('Locals', self.curframe.f_locals),
588 587 ('Globals', self.curframe.f_globals)]
589 588 self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
590 589
591 590 def do_pfile(self, arg):
592 591 """Print (or run through pager) the file where an object is defined.
593 592
594 593 The debugger interface to %pfile.
595 594 """
596 595 namespaces = [('Locals', self.curframe.f_locals),
597 596 ('Globals', self.curframe.f_globals)]
598 597 self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
599 598
600 599 def do_pinfo(self, arg):
601 600 """Provide detailed information about an object.
602 601
603 602 The debugger interface to %pinfo, i.e., obj?."""
604 603 namespaces = [('Locals', self.curframe.f_locals),
605 604 ('Globals', self.curframe.f_globals)]
606 605 self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
607 606
608 607 def do_pinfo2(self, arg):
609 608 """Provide extra detailed information about an object.
610 609
611 610 The debugger interface to %pinfo2, i.e., obj??."""
612 611 namespaces = [('Locals', self.curframe.f_locals),
613 612 ('Globals', self.curframe.f_globals)]
614 613 self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
615 614
616 615 def do_psource(self, arg):
617 616 """Print (or run through pager) the source code for an object."""
618 617 namespaces = [('Locals', self.curframe.f_locals),
619 618 ('Globals', self.curframe.f_globals)]
620 619 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
621 620
622 621 def do_where(self, arg):
623 622 """w(here)
624 623 Print a stack trace, with the most recent frame at the bottom.
625 624 An arrow indicates the "current frame", which determines the
626 625 context of most commands. 'bt' is an alias for this command.
627 626
628 627 Take a number as argument as an (optional) number of context line to
629 628 print"""
630 629 if arg:
631 630 context = int(arg)
632 631 self.print_stack_trace(context)
633 632 else:
634 633 self.print_stack_trace()
635 634
636 635 do_w = do_where
637 636
638 637
639 638 def set_trace(frame=None):
640 639 """
641 640 Start debugging from `frame`.
642 641
643 642 If frame is not specified, debugging starts from caller's frame.
644 643 """
645 644 Pdb().set_trace(frame or sys._getframe().f_back)
@@ -1,3659 +1,3658 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Main IPython class."""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 7 # Copyright (C) 2008-2011 The IPython Development Team
8 8 #
9 9 # Distributed under the terms of the BSD License. The full license is in
10 10 # the file COPYING, distributed as part of this software.
11 11 #-----------------------------------------------------------------------------
12 12
13 13
14 14 import abc
15 15 import ast
16 16 import asyncio
17 17 import atexit
18 18 import builtins as builtin_mod
19 19 import functools
20 20 import os
21 21 import re
22 22 import runpy
23 23 import sys
24 24 import tempfile
25 25 import traceback
26 26 import types
27 27 import subprocess
28 28 import warnings
29 29 from io import open as io_open
30 30
31 31 from pickleshare import PickleShareDB
32 32
33 33 from traitlets.config.configurable import SingletonConfigurable
34 34 from traitlets.utils.importstring import import_item
35 35 from IPython.core import oinspect
36 36 from IPython.core import magic
37 37 from IPython.core import page
38 38 from IPython.core import prefilter
39 39 from IPython.core import ultratb
40 40 from IPython.core.alias import Alias, AliasManager
41 41 from IPython.core.autocall import ExitAutocall
42 42 from IPython.core.builtin_trap import BuiltinTrap
43 43 from IPython.core.events import EventManager, available_events
44 44 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
45 45 from IPython.core.debugger import Pdb
46 46 from IPython.core.display_trap import DisplayTrap
47 47 from IPython.core.displayhook import DisplayHook
48 48 from IPython.core.displaypub import DisplayPublisher
49 49 from IPython.core.error import InputRejected, UsageError
50 50 from IPython.core.extensions import ExtensionManager
51 51 from IPython.core.formatters import DisplayFormatter
52 52 from IPython.core.history import HistoryManager
53 53 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
54 54 from IPython.core.logger import Logger
55 55 from IPython.core.macro import Macro
56 56 from IPython.core.payload import PayloadManager
57 57 from IPython.core.prefilter import PrefilterManager
58 58 from IPython.core.profiledir import ProfileDir
59 59 from IPython.core.usage import default_banner
60 60 from IPython.display import display
61 61 from IPython.testing.skipdoctest import skip_doctest
62 62 from IPython.utils import PyColorize
63 63 from IPython.utils import io
64 64 from IPython.utils import py3compat
65 65 from IPython.utils import openpy
66 66 from IPython.utils.decorators import undoc
67 67 from IPython.utils.io import ask_yes_no
68 68 from IPython.utils.ipstruct import Struct
69 69 from IPython.paths import get_ipython_dir
70 70 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
71 71 from IPython.utils.process import system, getoutput
72 72 from IPython.utils.strdispatch import StrDispatch
73 73 from IPython.utils.syspathcontext import prepended_to_syspath
74 74 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
75 75 from IPython.utils.tempdir import TemporaryDirectory
76 76 from traitlets import (
77 77 Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
78 78 observe, default, validate, Any
79 79 )
80 80 from warnings import warn
81 81 from logging import error
82 82 import IPython.core.hooks
83 83
84 84 from typing import List as ListType, Tuple
85 85 from ast import AST
86 86
87 87 # NoOpContext is deprecated, but ipykernel imports it from here.
88 88 # See https://github.com/ipython/ipykernel/issues/157
89 89 from IPython.utils.contexts import NoOpContext
90 90
91 91 try:
92 92 import docrepr.sphinxify as sphx
93 93
94 94 def sphinxify(doc):
95 95 with TemporaryDirectory() as dirname:
96 96 return {
97 97 'text/html': sphx.sphinxify(doc, dirname),
98 98 'text/plain': doc
99 99 }
100 100 except ImportError:
101 101 sphinxify = None
102 102
103 103
104 104 class ProvisionalWarning(DeprecationWarning):
105 105 """
106 106 Warning class for unstable features
107 107 """
108 108 pass
109 109
110 110 if sys.version_info > (3,6):
111 111 _assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign)
112 112 _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
113 113 else:
114 114 _assign_nodes = (ast.AugAssign, ast.Assign )
115 115 _single_targets_nodes = (ast.AugAssign, )
116 116
117 117 #-----------------------------------------------------------------------------
118 118 # Await Helpers
119 119 #-----------------------------------------------------------------------------
120 120
121 121 def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
122 122 """Return a function that do not create a new local scope.
123 123
124 124 Given a function, create a clone of this function where the co_newlocal flag
125 125 has been removed, making this function code actually run in the sourounding
126 126 scope.
127 127
128 128 We need this in order to run asynchronous code in user level namespace.
129 129 """
130 130 from types import CodeType, FunctionType
131 131 CO_NEWLOCALS = 0x0002
132 132 code = function.__code__
133 133 new_code = CodeType(
134 134 code.co_argcount,
135 135 code.co_kwonlyargcount,
136 136 code.co_nlocals,
137 137 code.co_stacksize,
138 138 code.co_flags & ~CO_NEWLOCALS,
139 139 code.co_code,
140 140 code.co_consts,
141 141 code.co_names,
142 142 code.co_varnames,
143 143 code.co_filename,
144 144 code.co_name,
145 145 code.co_firstlineno,
146 146 code.co_lnotab,
147 147 code.co_freevars,
148 148 code.co_cellvars
149 149 )
150 150 return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
151 151
152 152
153 153 # we still need to run things using the asyncio eventloop, but there is no
154 154 # async integration
155 155 from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner)
156 156
157 157 if sys.version_info > (3, 5):
158 158 from .async_helpers import _curio_runner, _trio_runner, _should_be_async
159 159 else :
160 160 _curio_runner = _trio_runner = None
161 161
162 162 def _should_be_async(cell:str)->bool:
163 163 return False
164 164
165 165
166 166 def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
167 167 """
168 168 Parse a cell with top-level await and modify the AST to be able to run it later.
169 169
170 170 Parameter
171 171 ---------
172 172
173 173 cell: str
174 174 The code cell to asyncronify
175 175 wrapper_name: str
176 176 The name of the function to be used to wrap the passed `cell`. It is
177 177 advised to **not** use a python identifier in order to not pollute the
178 178 global namespace in which the function will be ran.
179 179
180 180 Return
181 181 ------
182 182
183 183 A module object AST containing **one** function named `wrapper_name`.
184 184
185 185 The given code is wrapped in a async-def function, parsed into an AST, and
186 186 the resulting function definition AST is modified to return the last
187 187 expression.
188 188
189 189 The last expression or await node is moved into a return statement at the
190 190 end of the function, and removed from its original location. If the last
191 191 node is not Expr or Await nothing is done.
192 192
193 193 The function `__code__` will need to be later modified (by
194 194 ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
195 195 meaning that the local and global scope are the same, ie as if the body of
196 196 the function was at module level.
197 197
198 198 Lastly a call to `locals()` is made just before the last expression of the
199 199 function, or just after the last assignment or statement to make sure the
200 200 global dict is updated as python function work with a local fast cache which
201 201 is updated only on `local()` calls.
202 202 """
203 203
204 204 from ast import Expr, Await, Return
205 205 tree = ast.parse(_asyncify(cell))
206 206
207 207 function_def = tree.body[0]
208 208 function_def.name = wrapper_name
209 209 try_block = function_def.body[0]
210 210 lastexpr = try_block.body[-1]
211 211 if isinstance(lastexpr, (Expr, Await)):
212 212 try_block.body[-1] = Return(lastexpr.value)
213 213 ast.fix_missing_locations(tree)
214 214 return tree
215 215 #-----------------------------------------------------------------------------
216 216 # Globals
217 217 #-----------------------------------------------------------------------------
218 218
219 219 # compiled regexps for autoindent management
220 220 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
221 221
222 222 #-----------------------------------------------------------------------------
223 223 # Utilities
224 224 #-----------------------------------------------------------------------------
225 225
226 226 @undoc
227 227 def softspace(file, newvalue):
228 228 """Copied from code.py, to remove the dependency"""
229 229
230 230 oldvalue = 0
231 231 try:
232 232 oldvalue = file.softspace
233 233 except AttributeError:
234 234 pass
235 235 try:
236 236 file.softspace = newvalue
237 237 except (AttributeError, TypeError):
238 238 # "attribute-less object" or "read-only attributes"
239 239 pass
240 240 return oldvalue
241 241
242 242 @undoc
243 243 def no_op(*a, **kw):
244 244 pass
245 245
246 246
247 247 class SpaceInInput(Exception): pass
248 248
249 249
250 250 def get_default_colors():
251 251 "DEPRECATED"
252 252 warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.',
253 253 DeprecationWarning, stacklevel=2)
254 254 return 'Neutral'
255 255
256 256
257 257 class SeparateUnicode(Unicode):
258 258 r"""A Unicode subclass to validate separate_in, separate_out, etc.
259 259
260 260 This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
261 261 """
262 262
263 263 def validate(self, obj, value):
264 264 if value == '0': value = ''
265 265 value = value.replace('\\n','\n')
266 266 return super(SeparateUnicode, self).validate(obj, value)
267 267
268 268
269 269 @undoc
270 270 class DummyMod(object):
271 271 """A dummy module used for IPython's interactive module when
272 272 a namespace must be assigned to the module's __dict__."""
273 273 __spec__ = None
274 274
275 275
276 276 class ExecutionInfo(object):
277 277 """The arguments used for a call to :meth:`InteractiveShell.run_cell`
278 278
279 279 Stores information about what is going to happen.
280 280 """
281 281 raw_cell = None
282 282 store_history = False
283 283 silent = False
284 284 shell_futures = True
285 285
286 286 def __init__(self, raw_cell, store_history, silent, shell_futures):
287 287 self.raw_cell = raw_cell
288 288 self.store_history = store_history
289 289 self.silent = silent
290 290 self.shell_futures = shell_futures
291 291
292 292 def __repr__(self):
293 293 name = self.__class__.__qualname__
294 294 raw_cell = ((self.raw_cell[:50] + '..')
295 295 if len(self.raw_cell) > 50 else self.raw_cell)
296 296 return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s>' %\
297 297 (name, id(self), raw_cell, self.store_history, self.silent, self.shell_futures)
298 298
299 299
300 300 class ExecutionResult(object):
301 301 """The result of a call to :meth:`InteractiveShell.run_cell`
302 302
303 303 Stores information about what took place.
304 304 """
305 305 execution_count = None
306 306 error_before_exec = None
307 307 error_in_exec = None
308 308 info = None
309 309 result = None
310 310
311 311 def __init__(self, info):
312 312 self.info = info
313 313
314 314 @property
315 315 def success(self):
316 316 return (self.error_before_exec is None) and (self.error_in_exec is None)
317 317
318 318 def raise_error(self):
319 319 """Reraises error if `success` is `False`, otherwise does nothing"""
320 320 if self.error_before_exec is not None:
321 321 raise self.error_before_exec
322 322 if self.error_in_exec is not None:
323 323 raise self.error_in_exec
324 324
325 325 def __repr__(self):
326 326 name = self.__class__.__qualname__
327 327 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
328 328 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
329 329
330 330
331 331 class InteractiveShell(SingletonConfigurable):
332 332 """An enhanced, interactive shell for Python."""
333 333
334 334 _instance = None
335 335
336 336 ast_transformers = List([], help=
337 337 """
338 338 A list of ast.NodeTransformer subclass instances, which will be applied
339 339 to user input before code is run.
340 340 """
341 341 ).tag(config=True)
342 342
343 343 autocall = Enum((0,1,2), default_value=0, help=
344 344 """
345 345 Make IPython automatically call any callable object even if you didn't
346 346 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
347 347 automatically. The value can be '0' to disable the feature, '1' for
348 348 'smart' autocall, where it is not applied if there are no more
349 349 arguments on the line, and '2' for 'full' autocall, where all callable
350 350 objects are automatically called (even if no arguments are present).
351 351 """
352 352 ).tag(config=True)
353 353
354 354 autoindent = Bool(True, help=
355 355 """
356 356 Autoindent IPython code entered interactively.
357 357 """
358 358 ).tag(config=True)
359 359
360 360 autoawait = Bool(True, help=
361 361 """
362 362 Automatically run await statement in the top level repl.
363 363 """
364 364 ).tag(config=True)
365 365
366 366 loop_runner_map ={
367 367 'asyncio':(_asyncio_runner, True),
368 368 'curio':(_curio_runner, True),
369 369 'trio':(_trio_runner, True),
370 370 'sync': (_pseudo_sync_runner, False)
371 371 }
372 372
373 373 loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
374 374 allow_none=True,
375 375 help="""Select the loop runner that will be used to execute top-level asynchronous code"""
376 376 ).tag(config=True)
377 377
378 378 @default('loop_runner')
379 379 def _default_loop_runner(self):
380 380 return import_item("IPython.core.interactiveshell._asyncio_runner")
381 381
382 382 @validate('loop_runner')
383 383 def _import_runner(self, proposal):
384 384 if isinstance(proposal.value, str):
385 385 if proposal.value in self.loop_runner_map:
386 386 runner, autoawait = self.loop_runner_map[proposal.value]
387 387 self.autoawait = autoawait
388 388 return runner
389 389 runner = import_item(proposal.value)
390 390 if not callable(runner):
391 391 raise ValueError('loop_runner must be callable')
392 392 return runner
393 393 if not callable(proposal.value):
394 394 raise ValueError('loop_runner must be callable')
395 395 return proposal.value
396 396
397 397 automagic = Bool(True, help=
398 398 """
399 399 Enable magic commands to be called without the leading %.
400 400 """
401 401 ).tag(config=True)
402 402
403 403 banner1 = Unicode(default_banner,
404 404 help="""The part of the banner to be printed before the profile"""
405 405 ).tag(config=True)
406 406 banner2 = Unicode('',
407 407 help="""The part of the banner to be printed after the profile"""
408 408 ).tag(config=True)
409 409
410 410 cache_size = Integer(1000, help=
411 411 """
412 412 Set the size of the output cache. The default is 1000, you can
413 413 change it permanently in your config file. Setting it to 0 completely
414 414 disables the caching system, and the minimum value accepted is 3 (if
415 415 you provide a value less than 3, it is reset to 0 and a warning is
416 416 issued). This limit is defined because otherwise you'll spend more
417 417 time re-flushing a too small cache than working
418 418 """
419 419 ).tag(config=True)
420 420 color_info = Bool(True, help=
421 421 """
422 422 Use colors for displaying information about objects. Because this
423 423 information is passed through a pager (like 'less'), and some pagers
424 424 get confused with color codes, this capability can be turned off.
425 425 """
426 426 ).tag(config=True)
427 427 colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
428 428 default_value='Neutral',
429 429 help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
430 430 ).tag(config=True)
431 431 debug = Bool(False).tag(config=True)
432 432 disable_failing_post_execute = Bool(False,
433 433 help="Don't call post-execute functions that have failed in the past."
434 434 ).tag(config=True)
435 435 display_formatter = Instance(DisplayFormatter, allow_none=True)
436 436 displayhook_class = Type(DisplayHook)
437 437 display_pub_class = Type(DisplayPublisher)
438 438
439 439 sphinxify_docstring = Bool(False, help=
440 440 """
441 441 Enables rich html representation of docstrings. (This requires the
442 442 docrepr module).
443 443 """).tag(config=True)
444 444
445 445 @observe("sphinxify_docstring")
446 446 def _sphinxify_docstring_changed(self, change):
447 447 if change['new']:
448 448 warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
449 449
450 450 enable_html_pager = Bool(False, help=
451 451 """
452 452 (Provisional API) enables html representation in mime bundles sent
453 453 to pagers.
454 454 """).tag(config=True)
455 455
456 456 @observe("enable_html_pager")
457 457 def _enable_html_pager_changed(self, change):
458 458 if change['new']:
459 459 warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
460 460
461 461 data_pub_class = None
462 462
463 463 exit_now = Bool(False)
464 464 exiter = Instance(ExitAutocall)
465 465 @default('exiter')
466 466 def _exiter_default(self):
467 467 return ExitAutocall(self)
468 468 # Monotonically increasing execution counter
469 469 execution_count = Integer(1)
470 470 filename = Unicode("<ipython console>")
471 471 ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
472 472
473 473 # Used to transform cells before running them, and check whether code is complete
474 474 input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
475 475 ())
476 476
477 477 @property
478 478 def input_transformers_cleanup(self):
479 479 return self.input_transformer_manager.cleanup_transforms
480 480
481 481 input_transformers_post = List([],
482 482 help="A list of string input transformers, to be applied after IPython's "
483 483 "own input transformations."
484 484 )
485 485
486 486 @property
487 487 def input_splitter(self):
488 488 """Make this available for backward compatibility (pre-7.0 release) with existing code.
489 489
490 490 For example, ipykernel ipykernel currently uses
491 491 `shell.input_splitter.check_complete`
492 492 """
493 493 from warnings import warn
494 494 warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
495 495 DeprecationWarning, stacklevel=2
496 496 )
497 497 return self.input_transformer_manager
498 498
499 499 logstart = Bool(False, help=
500 500 """
501 501 Start logging to the default log file in overwrite mode.
502 502 Use `logappend` to specify a log file to **append** logs to.
503 503 """
504 504 ).tag(config=True)
505 505 logfile = Unicode('', help=
506 506 """
507 507 The name of the logfile to use.
508 508 """
509 509 ).tag(config=True)
510 510 logappend = Unicode('', help=
511 511 """
512 512 Start logging to the given file in append mode.
513 513 Use `logfile` to specify a log file to **overwrite** logs to.
514 514 """
515 515 ).tag(config=True)
516 516 object_info_string_level = Enum((0,1,2), default_value=0,
517 517 ).tag(config=True)
518 518 pdb = Bool(False, help=
519 519 """
520 520 Automatically call the pdb debugger after every exception.
521 521 """
522 522 ).tag(config=True)
523 523 display_page = Bool(False,
524 524 help="""If True, anything that would be passed to the pager
525 525 will be displayed as regular output instead."""
526 526 ).tag(config=True)
527 527
528 528 # deprecated prompt traits:
529 529
530 530 prompt_in1 = Unicode('In [\\#]: ',
531 531 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
532 532 ).tag(config=True)
533 533 prompt_in2 = Unicode(' .\\D.: ',
534 534 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
535 535 ).tag(config=True)
536 536 prompt_out = Unicode('Out[\\#]: ',
537 537 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
538 538 ).tag(config=True)
539 539 prompts_pad_left = Bool(True,
540 540 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
541 541 ).tag(config=True)
542 542
543 543 @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
544 544 def _prompt_trait_changed(self, change):
545 545 name = change['name']
546 546 warn("InteractiveShell.{name} is deprecated since IPython 4.0"
547 547 " and ignored since 5.0, set TerminalInteractiveShell.prompts"
548 548 " object directly.".format(name=name))
549 549
550 550 # protect against weird cases where self.config may not exist:
551 551
552 552 show_rewritten_input = Bool(True,
553 553 help="Show rewritten input, e.g. for autocall."
554 554 ).tag(config=True)
555 555
556 556 quiet = Bool(False).tag(config=True)
557 557
558 558 history_length = Integer(10000,
559 559 help='Total length of command history'
560 560 ).tag(config=True)
561 561
562 562 history_load_length = Integer(1000, help=
563 563 """
564 564 The number of saved history entries to be loaded
565 565 into the history buffer at startup.
566 566 """
567 567 ).tag(config=True)
568 568
569 569 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
570 570 default_value='last_expr',
571 571 help="""
572 572 'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
573 573 which nodes should be run interactively (displaying output from expressions).
574 574 """
575 575 ).tag(config=True)
576 576
577 577 # TODO: this part of prompt management should be moved to the frontends.
578 578 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
579 579 separate_in = SeparateUnicode('\n').tag(config=True)
580 580 separate_out = SeparateUnicode('').tag(config=True)
581 581 separate_out2 = SeparateUnicode('').tag(config=True)
582 582 wildcards_case_sensitive = Bool(True).tag(config=True)
583 583 xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
584 584 default_value='Context',
585 585 help="Switch modes for the IPython exception handlers."
586 586 ).tag(config=True)
587 587
588 588 # Subcomponents of InteractiveShell
589 589 alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
590 590 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
591 591 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
592 592 display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
593 593 extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
594 594 payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
595 595 history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
596 596 magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
597 597
598 598 profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
599 599 @property
600 600 def profile(self):
601 601 if self.profile_dir is not None:
602 602 name = os.path.basename(self.profile_dir.location)
603 603 return name.replace('profile_','')
604 604
605 605
606 606 # Private interface
607 607 _post_execute = Dict()
608 608
609 609 # Tracks any GUI loop loaded for pylab
610 610 pylab_gui_select = None
611 611
612 612 last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
613 613
614 614 last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
615 615
616 616 def __init__(self, ipython_dir=None, profile_dir=None,
617 617 user_module=None, user_ns=None,
618 618 custom_exceptions=((), None), **kwargs):
619 619
620 620 # This is where traits with a config_key argument are updated
621 621 # from the values on config.
622 622 super(InteractiveShell, self).__init__(**kwargs)
623 623 if 'PromptManager' in self.config:
624 624 warn('As of IPython 5.0 `PromptManager` config will have no effect'
625 625 ' and has been replaced by TerminalInteractiveShell.prompts_class')
626 626 self.configurables = [self]
627 627
628 628 # These are relatively independent and stateless
629 629 self.init_ipython_dir(ipython_dir)
630 630 self.init_profile_dir(profile_dir)
631 631 self.init_instance_attrs()
632 632 self.init_environment()
633 633
634 634 # Check if we're in a virtualenv, and set up sys.path.
635 635 self.init_virtualenv()
636 636
637 637 # Create namespaces (user_ns, user_global_ns, etc.)
638 638 self.init_create_namespaces(user_module, user_ns)
639 639 # This has to be done after init_create_namespaces because it uses
640 640 # something in self.user_ns, but before init_sys_modules, which
641 641 # is the first thing to modify sys.
642 642 # TODO: When we override sys.stdout and sys.stderr before this class
643 643 # is created, we are saving the overridden ones here. Not sure if this
644 644 # is what we want to do.
645 645 self.save_sys_module_state()
646 646 self.init_sys_modules()
647 647
648 648 # While we're trying to have each part of the code directly access what
649 649 # it needs without keeping redundant references to objects, we have too
650 650 # much legacy code that expects ip.db to exist.
651 651 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
652 652
653 653 self.init_history()
654 654 self.init_encoding()
655 655 self.init_prefilter()
656 656
657 657 self.init_syntax_highlighting()
658 658 self.init_hooks()
659 659 self.init_events()
660 660 self.init_pushd_popd_magic()
661 661 self.init_user_ns()
662 662 self.init_logger()
663 663 self.init_builtins()
664 664
665 665 # The following was in post_config_initialization
666 666 self.init_inspector()
667 667 self.raw_input_original = input
668 668 self.init_completer()
669 669 # TODO: init_io() needs to happen before init_traceback handlers
670 670 # because the traceback handlers hardcode the stdout/stderr streams.
671 671 # This logic in in debugger.Pdb and should eventually be changed.
672 672 self.init_io()
673 673 self.init_traceback_handlers(custom_exceptions)
674 674 self.init_prompts()
675 675 self.init_display_formatter()
676 676 self.init_display_pub()
677 677 self.init_data_pub()
678 678 self.init_displayhook()
679 679 self.init_magics()
680 680 self.init_alias()
681 681 self.init_logstart()
682 682 self.init_pdb()
683 683 self.init_extension_manager()
684 684 self.init_payload()
685 685 self.init_deprecation_warnings()
686 686 self.hooks.late_startup_hook()
687 687 self.events.trigger('shell_initialized', self)
688 688 atexit.register(self.atexit_operations)
689 689
690 690 def get_ipython(self):
691 691 """Return the currently running IPython instance."""
692 692 return self
693 693
694 694 #-------------------------------------------------------------------------
695 695 # Trait changed handlers
696 696 #-------------------------------------------------------------------------
697 697 @observe('ipython_dir')
698 698 def _ipython_dir_changed(self, change):
699 699 ensure_dir_exists(change['new'])
700 700
701 701 def set_autoindent(self,value=None):
702 702 """Set the autoindent flag.
703 703
704 704 If called with no arguments, it acts as a toggle."""
705 705 if value is None:
706 706 self.autoindent = not self.autoindent
707 707 else:
708 708 self.autoindent = value
709 709
710 710 #-------------------------------------------------------------------------
711 711 # init_* methods called by __init__
712 712 #-------------------------------------------------------------------------
713 713
714 714 def init_ipython_dir(self, ipython_dir):
715 715 if ipython_dir is not None:
716 716 self.ipython_dir = ipython_dir
717 717 return
718 718
719 719 self.ipython_dir = get_ipython_dir()
720 720
721 721 def init_profile_dir(self, profile_dir):
722 722 if profile_dir is not None:
723 723 self.profile_dir = profile_dir
724 724 return
725 725 self.profile_dir =\
726 726 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
727 727
728 728 def init_instance_attrs(self):
729 729 self.more = False
730 730
731 731 # command compiler
732 732 self.compile = CachingCompiler()
733 733
734 734 # Make an empty namespace, which extension writers can rely on both
735 735 # existing and NEVER being used by ipython itself. This gives them a
736 736 # convenient location for storing additional information and state
737 737 # their extensions may require, without fear of collisions with other
738 738 # ipython names that may develop later.
739 739 self.meta = Struct()
740 740
741 741 # Temporary files used for various purposes. Deleted at exit.
742 742 self.tempfiles = []
743 743 self.tempdirs = []
744 744
745 745 # keep track of where we started running (mainly for crash post-mortem)
746 746 # This is not being used anywhere currently.
747 747 self.starting_dir = os.getcwd()
748 748
749 749 # Indentation management
750 750 self.indent_current_nsp = 0
751 751
752 752 # Dict to track post-execution functions that have been registered
753 753 self._post_execute = {}
754 754
755 755 def init_environment(self):
756 756 """Any changes we need to make to the user's environment."""
757 757 pass
758 758
759 759 def init_encoding(self):
760 760 # Get system encoding at startup time. Certain terminals (like Emacs
761 761 # under Win32 have it set to None, and we need to have a known valid
762 762 # encoding to use in the raw_input() method
763 763 try:
764 764 self.stdin_encoding = sys.stdin.encoding or 'ascii'
765 765 except AttributeError:
766 766 self.stdin_encoding = 'ascii'
767 767
768 768
769 769 @observe('colors')
770 770 def init_syntax_highlighting(self, changes=None):
771 771 # Python source parser/formatter for syntax highlighting
772 772 pyformat = PyColorize.Parser(style=self.colors, parent=self).format
773 773 self.pycolorize = lambda src: pyformat(src,'str')
774 774
775 775 def refresh_style(self):
776 776 # No-op here, used in subclass
777 777 pass
778 778
779 779 def init_pushd_popd_magic(self):
780 780 # for pushd/popd management
781 781 self.home_dir = get_home_dir()
782 782
783 783 self.dir_stack = []
784 784
785 785 def init_logger(self):
786 786 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
787 787 logmode='rotate')
788 788
789 789 def init_logstart(self):
790 790 """Initialize logging in case it was requested at the command line.
791 791 """
792 792 if self.logappend:
793 793 self.magic('logstart %s append' % self.logappend)
794 794 elif self.logfile:
795 795 self.magic('logstart %s' % self.logfile)
796 796 elif self.logstart:
797 797 self.magic('logstart')
798 798
799 799 def init_deprecation_warnings(self):
800 800 """
801 801 register default filter for deprecation warning.
802 802
803 803 This will allow deprecation warning of function used interactively to show
804 804 warning to users, and still hide deprecation warning from libraries import.
805 805 """
806 806 if sys.version_info < (3,7):
807 807 warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
808 808
809 809
810 810 def init_builtins(self):
811 811 # A single, static flag that we set to True. Its presence indicates
812 812 # that an IPython shell has been created, and we make no attempts at
813 813 # removing on exit or representing the existence of more than one
814 814 # IPython at a time.
815 815 builtin_mod.__dict__['__IPYTHON__'] = True
816 816 builtin_mod.__dict__['display'] = display
817 817
818 818 self.builtin_trap = BuiltinTrap(shell=self)
819 819
820 820 @observe('colors')
821 821 def init_inspector(self, changes=None):
822 822 # Object inspector
823 823 self.inspector = oinspect.Inspector(oinspect.InspectColors,
824 824 PyColorize.ANSICodeColors,
825 825 self.colors,
826 826 self.object_info_string_level)
827 827
828 828 def init_io(self):
829 829 # This will just use sys.stdout and sys.stderr. If you want to
830 830 # override sys.stdout and sys.stderr themselves, you need to do that
831 831 # *before* instantiating this class, because io holds onto
832 832 # references to the underlying streams.
833 833 # io.std* are deprecated, but don't show our own deprecation warnings
834 834 # during initialization of the deprecated API.
835 835 with warnings.catch_warnings():
836 836 warnings.simplefilter('ignore', DeprecationWarning)
837 837 io.stdout = io.IOStream(sys.stdout)
838 838 io.stderr = io.IOStream(sys.stderr)
839 839
840 840 def init_prompts(self):
841 841 # Set system prompts, so that scripts can decide if they are running
842 842 # interactively.
843 843 sys.ps1 = 'In : '
844 844 sys.ps2 = '...: '
845 845 sys.ps3 = 'Out: '
846 846
847 847 def init_display_formatter(self):
848 848 self.display_formatter = DisplayFormatter(parent=self)
849 849 self.configurables.append(self.display_formatter)
850 850
851 851 def init_display_pub(self):
852 852 self.display_pub = self.display_pub_class(parent=self)
853 853 self.configurables.append(self.display_pub)
854 854
855 855 def init_data_pub(self):
856 856 if not self.data_pub_class:
857 857 self.data_pub = None
858 858 return
859 859 self.data_pub = self.data_pub_class(parent=self)
860 860 self.configurables.append(self.data_pub)
861 861
862 862 def init_displayhook(self):
863 863 # Initialize displayhook, set in/out prompts and printing system
864 864 self.displayhook = self.displayhook_class(
865 865 parent=self,
866 866 shell=self,
867 867 cache_size=self.cache_size,
868 868 )
869 869 self.configurables.append(self.displayhook)
870 870 # This is a context manager that installs/revmoes the displayhook at
871 871 # the appropriate time.
872 872 self.display_trap = DisplayTrap(hook=self.displayhook)
873 873
874 874 def init_virtualenv(self):
875 875 """Add a virtualenv to sys.path so the user can import modules from it.
876 876 This isn't perfect: it doesn't use the Python interpreter with which the
877 877 virtualenv was built, and it ignores the --no-site-packages option. A
878 878 warning will appear suggesting the user installs IPython in the
879 879 virtualenv, but for many cases, it probably works well enough.
880 880
881 881 Adapted from code snippets online.
882 882
883 883 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
884 884 """
885 885 if 'VIRTUAL_ENV' not in os.environ:
886 886 # Not in a virtualenv
887 887 return
888 888
889 889 p = os.path.normcase(sys.executable)
890 890 p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
891 891
892 892 # executable path should end like /bin/python or \\scripts\\python.exe
893 893 p_exe_up2 = os.path.dirname(os.path.dirname(p))
894 894 if p_exe_up2 and os.path.exists(p_venv) and os.path.samefile(p_exe_up2, p_venv):
895 895 # Our exe is inside the virtualenv, don't need to do anything.
896 896 return
897 897
898 898 # fallback venv detection:
899 899 # stdlib venv may symlink sys.executable, so we can't use realpath.
900 900 # but others can symlink *to* the venv Python, so we can't just use sys.executable.
901 901 # So we just check every item in the symlink tree (generally <= 3)
902 902 paths = [p]
903 903 while os.path.islink(p):
904 904 p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
905 905 paths.append(p)
906 906
907 907 # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
908 908 if p_venv.startswith('\\cygdrive'):
909 909 p_venv = p_venv[11:]
910 910 elif len(p_venv) >= 2 and p_venv[1] == ':':
911 911 p_venv = p_venv[2:]
912 912
913 913 if any(p_venv in p for p in paths):
914 914 # Running properly in the virtualenv, don't need to do anything
915 915 return
916 916
917 917 warn("Attempting to work in a virtualenv. If you encounter problems, please "
918 918 "install IPython inside the virtualenv.")
919 919 if sys.platform == "win32":
920 920 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
921 921 else:
922 922 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
923 923 'python%d.%d' % sys.version_info[:2], 'site-packages')
924 924
925 925 import site
926 926 sys.path.insert(0, virtual_env)
927 927 site.addsitedir(virtual_env)
928 928
929 929 #-------------------------------------------------------------------------
930 930 # Things related to injections into the sys module
931 931 #-------------------------------------------------------------------------
932 932
933 933 def save_sys_module_state(self):
934 934 """Save the state of hooks in the sys module.
935 935
936 936 This has to be called after self.user_module is created.
937 937 """
938 938 self._orig_sys_module_state = {'stdin': sys.stdin,
939 939 'stdout': sys.stdout,
940 940 'stderr': sys.stderr,
941 941 'excepthook': sys.excepthook}
942 942 self._orig_sys_modules_main_name = self.user_module.__name__
943 943 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
944 944
945 945 def restore_sys_module_state(self):
946 946 """Restore the state of the sys module."""
947 947 try:
948 948 for k, v in self._orig_sys_module_state.items():
949 949 setattr(sys, k, v)
950 950 except AttributeError:
951 951 pass
952 952 # Reset what what done in self.init_sys_modules
953 953 if self._orig_sys_modules_main_mod is not None:
954 954 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
955 955
956 956 #-------------------------------------------------------------------------
957 957 # Things related to the banner
958 958 #-------------------------------------------------------------------------
959 959
960 960 @property
961 961 def banner(self):
962 962 banner = self.banner1
963 963 if self.profile and self.profile != 'default':
964 964 banner += '\nIPython profile: %s\n' % self.profile
965 965 if self.banner2:
966 966 banner += '\n' + self.banner2
967 967 return banner
968 968
969 969 def show_banner(self, banner=None):
970 970 if banner is None:
971 971 banner = self.banner
972 972 sys.stdout.write(banner)
973 973
974 974 #-------------------------------------------------------------------------
975 975 # Things related to hooks
976 976 #-------------------------------------------------------------------------
977 977
978 978 def init_hooks(self):
979 979 # hooks holds pointers used for user-side customizations
980 980 self.hooks = Struct()
981 981
982 982 self.strdispatchers = {}
983 983
984 984 # Set all default hooks, defined in the IPython.hooks module.
985 985 hooks = IPython.core.hooks
986 986 for hook_name in hooks.__all__:
987 987 # default hooks have priority 100, i.e. low; user hooks should have
988 988 # 0-100 priority
989 989 self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
990 990
991 991 if self.display_page:
992 992 self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
993 993
994 994 def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
995 995 _warn_deprecated=True):
996 996 """set_hook(name,hook) -> sets an internal IPython hook.
997 997
998 998 IPython exposes some of its internal API as user-modifiable hooks. By
999 999 adding your function to one of these hooks, you can modify IPython's
1000 1000 behavior to call at runtime your own routines."""
1001 1001
1002 1002 # At some point in the future, this should validate the hook before it
1003 1003 # accepts it. Probably at least check that the hook takes the number
1004 1004 # of args it's supposed to.
1005 1005
1006 1006 f = types.MethodType(hook,self)
1007 1007
1008 1008 # check if the hook is for strdispatcher first
1009 1009 if str_key is not None:
1010 1010 sdp = self.strdispatchers.get(name, StrDispatch())
1011 1011 sdp.add_s(str_key, f, priority )
1012 1012 self.strdispatchers[name] = sdp
1013 1013 return
1014 1014 if re_key is not None:
1015 1015 sdp = self.strdispatchers.get(name, StrDispatch())
1016 1016 sdp.add_re(re.compile(re_key), f, priority )
1017 1017 self.strdispatchers[name] = sdp
1018 1018 return
1019 1019
1020 1020 dp = getattr(self.hooks, name, None)
1021 1021 if name not in IPython.core.hooks.__all__:
1022 1022 print("Warning! Hook '%s' is not one of %s" % \
1023 1023 (name, IPython.core.hooks.__all__ ))
1024 1024
1025 1025 if _warn_deprecated and (name in IPython.core.hooks.deprecated):
1026 1026 alternative = IPython.core.hooks.deprecated[name]
1027 1027 warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
1028 1028
1029 1029 if not dp:
1030 1030 dp = IPython.core.hooks.CommandChainDispatcher()
1031 1031
1032 1032 try:
1033 1033 dp.add(f,priority)
1034 1034 except AttributeError:
1035 1035 # it was not commandchain, plain old func - replace
1036 1036 dp = f
1037 1037
1038 1038 setattr(self.hooks,name, dp)
1039 1039
1040 1040 #-------------------------------------------------------------------------
1041 1041 # Things related to events
1042 1042 #-------------------------------------------------------------------------
1043 1043
1044 1044 def init_events(self):
1045 1045 self.events = EventManager(self, available_events)
1046 1046
1047 1047 self.events.register("pre_execute", self._clear_warning_registry)
1048 1048
1049 1049 def register_post_execute(self, func):
1050 1050 """DEPRECATED: Use ip.events.register('post_run_cell', func)
1051 1051
1052 1052 Register a function for calling after code execution.
1053 1053 """
1054 1054 warn("ip.register_post_execute is deprecated, use "
1055 1055 "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
1056 1056 self.events.register('post_run_cell', func)
1057 1057
1058 1058 def _clear_warning_registry(self):
1059 1059 # clear the warning registry, so that different code blocks with
1060 1060 # overlapping line number ranges don't cause spurious suppression of
1061 1061 # warnings (see gh-6611 for details)
1062 1062 if "__warningregistry__" in self.user_global_ns:
1063 1063 del self.user_global_ns["__warningregistry__"]
1064 1064
1065 1065 #-------------------------------------------------------------------------
1066 1066 # Things related to the "main" module
1067 1067 #-------------------------------------------------------------------------
1068 1068
1069 1069 def new_main_mod(self, filename, modname):
1070 1070 """Return a new 'main' module object for user code execution.
1071 1071
1072 1072 ``filename`` should be the path of the script which will be run in the
1073 1073 module. Requests with the same filename will get the same module, with
1074 1074 its namespace cleared.
1075 1075
1076 1076 ``modname`` should be the module name - normally either '__main__' or
1077 1077 the basename of the file without the extension.
1078 1078
1079 1079 When scripts are executed via %run, we must keep a reference to their
1080 1080 __main__ module around so that Python doesn't
1081 1081 clear it, rendering references to module globals useless.
1082 1082
1083 1083 This method keeps said reference in a private dict, keyed by the
1084 1084 absolute path of the script. This way, for multiple executions of the
1085 1085 same script we only keep one copy of the namespace (the last one),
1086 1086 thus preventing memory leaks from old references while allowing the
1087 1087 objects from the last execution to be accessible.
1088 1088 """
1089 1089 filename = os.path.abspath(filename)
1090 1090 try:
1091 1091 main_mod = self._main_mod_cache[filename]
1092 1092 except KeyError:
1093 1093 main_mod = self._main_mod_cache[filename] = types.ModuleType(
1094 1094 modname,
1095 1095 doc="Module created for script run in IPython")
1096 1096 else:
1097 1097 main_mod.__dict__.clear()
1098 1098 main_mod.__name__ = modname
1099 1099
1100 1100 main_mod.__file__ = filename
1101 1101 # It seems pydoc (and perhaps others) needs any module instance to
1102 1102 # implement a __nonzero__ method
1103 1103 main_mod.__nonzero__ = lambda : True
1104 1104
1105 1105 return main_mod
1106 1106
1107 1107 def clear_main_mod_cache(self):
1108 1108 """Clear the cache of main modules.
1109 1109
1110 1110 Mainly for use by utilities like %reset.
1111 1111
1112 1112 Examples
1113 1113 --------
1114 1114
1115 1115 In [15]: import IPython
1116 1116
1117 1117 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
1118 1118
1119 1119 In [17]: len(_ip._main_mod_cache) > 0
1120 1120 Out[17]: True
1121 1121
1122 1122 In [18]: _ip.clear_main_mod_cache()
1123 1123
1124 1124 In [19]: len(_ip._main_mod_cache) == 0
1125 1125 Out[19]: True
1126 1126 """
1127 1127 self._main_mod_cache.clear()
1128 1128
1129 1129 #-------------------------------------------------------------------------
1130 1130 # Things related to debugging
1131 1131 #-------------------------------------------------------------------------
1132 1132
1133 1133 def init_pdb(self):
1134 1134 # Set calling of pdb on exceptions
1135 1135 # self.call_pdb is a property
1136 1136 self.call_pdb = self.pdb
1137 1137
1138 1138 def _get_call_pdb(self):
1139 1139 return self._call_pdb
1140 1140
1141 1141 def _set_call_pdb(self,val):
1142 1142
1143 1143 if val not in (0,1,False,True):
1144 1144 raise ValueError('new call_pdb value must be boolean')
1145 1145
1146 1146 # store value in instance
1147 1147 self._call_pdb = val
1148 1148
1149 1149 # notify the actual exception handlers
1150 1150 self.InteractiveTB.call_pdb = val
1151 1151
1152 1152 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
1153 1153 'Control auto-activation of pdb at exceptions')
1154 1154
1155 1155 def debugger(self,force=False):
1156 1156 """Call the pdb debugger.
1157 1157
1158 1158 Keywords:
1159 1159
1160 1160 - force(False): by default, this routine checks the instance call_pdb
1161 1161 flag and does not actually invoke the debugger if the flag is false.
1162 1162 The 'force' option forces the debugger to activate even if the flag
1163 1163 is false.
1164 1164 """
1165 1165
1166 1166 if not (force or self.call_pdb):
1167 1167 return
1168 1168
1169 1169 if not hasattr(sys,'last_traceback'):
1170 1170 error('No traceback has been produced, nothing to debug.')
1171 1171 return
1172 1172
1173 1173 self.InteractiveTB.debugger(force=True)
1174 1174
1175 1175 #-------------------------------------------------------------------------
1176 1176 # Things related to IPython's various namespaces
1177 1177 #-------------------------------------------------------------------------
1178 1178 default_user_namespaces = True
1179 1179
1180 1180 def init_create_namespaces(self, user_module=None, user_ns=None):
1181 1181 # Create the namespace where the user will operate. user_ns is
1182 1182 # normally the only one used, and it is passed to the exec calls as
1183 1183 # the locals argument. But we do carry a user_global_ns namespace
1184 1184 # given as the exec 'globals' argument, This is useful in embedding
1185 1185 # situations where the ipython shell opens in a context where the
1186 1186 # distinction between locals and globals is meaningful. For
1187 1187 # non-embedded contexts, it is just the same object as the user_ns dict.
1188 1188
1189 1189 # FIXME. For some strange reason, __builtins__ is showing up at user
1190 1190 # level as a dict instead of a module. This is a manual fix, but I
1191 1191 # should really track down where the problem is coming from. Alex
1192 1192 # Schmolck reported this problem first.
1193 1193
1194 1194 # A useful post by Alex Martelli on this topic:
1195 1195 # Re: inconsistent value from __builtins__
1196 1196 # Von: Alex Martelli <aleaxit@yahoo.com>
1197 1197 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
1198 1198 # Gruppen: comp.lang.python
1199 1199
1200 1200 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
1201 1201 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
1202 1202 # > <type 'dict'>
1203 1203 # > >>> print type(__builtins__)
1204 1204 # > <type 'module'>
1205 1205 # > Is this difference in return value intentional?
1206 1206
1207 1207 # Well, it's documented that '__builtins__' can be either a dictionary
1208 1208 # or a module, and it's been that way for a long time. Whether it's
1209 1209 # intentional (or sensible), I don't know. In any case, the idea is
1210 1210 # that if you need to access the built-in namespace directly, you
1211 1211 # should start with "import __builtin__" (note, no 's') which will
1212 1212 # definitely give you a module. Yeah, it's somewhat confusing:-(.
1213 1213
1214 1214 # These routines return a properly built module and dict as needed by
1215 1215 # the rest of the code, and can also be used by extension writers to
1216 1216 # generate properly initialized namespaces.
1217 1217 if (user_ns is not None) or (user_module is not None):
1218 1218 self.default_user_namespaces = False
1219 1219 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
1220 1220
1221 1221 # A record of hidden variables we have added to the user namespace, so
1222 1222 # we can list later only variables defined in actual interactive use.
1223 1223 self.user_ns_hidden = {}
1224 1224
1225 1225 # Now that FakeModule produces a real module, we've run into a nasty
1226 1226 # problem: after script execution (via %run), the module where the user
1227 1227 # code ran is deleted. Now that this object is a true module (needed
1228 1228 # so doctest and other tools work correctly), the Python module
1229 1229 # teardown mechanism runs over it, and sets to None every variable
1230 1230 # present in that module. Top-level references to objects from the
1231 1231 # script survive, because the user_ns is updated with them. However,
1232 1232 # calling functions defined in the script that use other things from
1233 1233 # the script will fail, because the function's closure had references
1234 1234 # to the original objects, which are now all None. So we must protect
1235 1235 # these modules from deletion by keeping a cache.
1236 1236 #
1237 1237 # To avoid keeping stale modules around (we only need the one from the
1238 1238 # last run), we use a dict keyed with the full path to the script, so
1239 1239 # only the last version of the module is held in the cache. Note,
1240 1240 # however, that we must cache the module *namespace contents* (their
1241 1241 # __dict__). Because if we try to cache the actual modules, old ones
1242 1242 # (uncached) could be destroyed while still holding references (such as
1243 1243 # those held by GUI objects that tend to be long-lived)>
1244 1244 #
1245 1245 # The %reset command will flush this cache. See the cache_main_mod()
1246 1246 # and clear_main_mod_cache() methods for details on use.
1247 1247
1248 1248 # This is the cache used for 'main' namespaces
1249 1249 self._main_mod_cache = {}
1250 1250
1251 1251 # A table holding all the namespaces IPython deals with, so that
1252 1252 # introspection facilities can search easily.
1253 1253 self.ns_table = {'user_global':self.user_module.__dict__,
1254 1254 'user_local':self.user_ns,
1255 1255 'builtin':builtin_mod.__dict__
1256 1256 }
1257 1257
1258 1258 @property
1259 1259 def user_global_ns(self):
1260 1260 return self.user_module.__dict__
1261 1261
1262 1262 def prepare_user_module(self, user_module=None, user_ns=None):
1263 1263 """Prepare the module and namespace in which user code will be run.
1264 1264
1265 1265 When IPython is started normally, both parameters are None: a new module
1266 1266 is created automatically, and its __dict__ used as the namespace.
1267 1267
1268 1268 If only user_module is provided, its __dict__ is used as the namespace.
1269 1269 If only user_ns is provided, a dummy module is created, and user_ns
1270 1270 becomes the global namespace. If both are provided (as they may be
1271 1271 when embedding), user_ns is the local namespace, and user_module
1272 1272 provides the global namespace.
1273 1273
1274 1274 Parameters
1275 1275 ----------
1276 1276 user_module : module, optional
1277 1277 The current user module in which IPython is being run. If None,
1278 1278 a clean module will be created.
1279 1279 user_ns : dict, optional
1280 1280 A namespace in which to run interactive commands.
1281 1281
1282 1282 Returns
1283 1283 -------
1284 1284 A tuple of user_module and user_ns, each properly initialised.
1285 1285 """
1286 1286 if user_module is None and user_ns is not None:
1287 1287 user_ns.setdefault("__name__", "__main__")
1288 1288 user_module = DummyMod()
1289 1289 user_module.__dict__ = user_ns
1290 1290
1291 1291 if user_module is None:
1292 1292 user_module = types.ModuleType("__main__",
1293 1293 doc="Automatically created module for IPython interactive environment")
1294 1294
1295 1295 # We must ensure that __builtin__ (without the final 's') is always
1296 1296 # available and pointing to the __builtin__ *module*. For more details:
1297 1297 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1298 1298 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1299 1299 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1300 1300
1301 1301 if user_ns is None:
1302 1302 user_ns = user_module.__dict__
1303 1303
1304 1304 return user_module, user_ns
1305 1305
1306 1306 def init_sys_modules(self):
1307 1307 # We need to insert into sys.modules something that looks like a
1308 1308 # module but which accesses the IPython namespace, for shelve and
1309 1309 # pickle to work interactively. Normally they rely on getting
1310 1310 # everything out of __main__, but for embedding purposes each IPython
1311 1311 # instance has its own private namespace, so we can't go shoving
1312 1312 # everything into __main__.
1313 1313
1314 1314 # note, however, that we should only do this for non-embedded
1315 1315 # ipythons, which really mimic the __main__.__dict__ with their own
1316 1316 # namespace. Embedded instances, on the other hand, should not do
1317 1317 # this because they need to manage the user local/global namespaces
1318 1318 # only, but they live within a 'normal' __main__ (meaning, they
1319 1319 # shouldn't overtake the execution environment of the script they're
1320 1320 # embedded in).
1321 1321
1322 1322 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1323 1323 main_name = self.user_module.__name__
1324 1324 sys.modules[main_name] = self.user_module
1325 1325
1326 1326 def init_user_ns(self):
1327 1327 """Initialize all user-visible namespaces to their minimum defaults.
1328 1328
1329 1329 Certain history lists are also initialized here, as they effectively
1330 1330 act as user namespaces.
1331 1331
1332 1332 Notes
1333 1333 -----
1334 1334 All data structures here are only filled in, they are NOT reset by this
1335 1335 method. If they were not empty before, data will simply be added to
1336 1336 them.
1337 1337 """
1338 1338 # This function works in two parts: first we put a few things in
1339 1339 # user_ns, and we sync that contents into user_ns_hidden so that these
1340 1340 # initial variables aren't shown by %who. After the sync, we add the
1341 1341 # rest of what we *do* want the user to see with %who even on a new
1342 1342 # session (probably nothing, so they really only see their own stuff)
1343 1343
1344 1344 # The user dict must *always* have a __builtin__ reference to the
1345 1345 # Python standard __builtin__ namespace, which must be imported.
1346 1346 # This is so that certain operations in prompt evaluation can be
1347 1347 # reliably executed with builtins. Note that we can NOT use
1348 1348 # __builtins__ (note the 's'), because that can either be a dict or a
1349 1349 # module, and can even mutate at runtime, depending on the context
1350 1350 # (Python makes no guarantees on it). In contrast, __builtin__ is
1351 1351 # always a module object, though it must be explicitly imported.
1352 1352
1353 1353 # For more details:
1354 1354 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1355 1355 ns = {}
1356 1356
1357 1357 # make global variables for user access to the histories
1358 1358 ns['_ih'] = self.history_manager.input_hist_parsed
1359 1359 ns['_oh'] = self.history_manager.output_hist
1360 1360 ns['_dh'] = self.history_manager.dir_hist
1361 1361
1362 1362 # user aliases to input and output histories. These shouldn't show up
1363 1363 # in %who, as they can have very large reprs.
1364 1364 ns['In'] = self.history_manager.input_hist_parsed
1365 1365 ns['Out'] = self.history_manager.output_hist
1366 1366
1367 1367 # Store myself as the public api!!!
1368 1368 ns['get_ipython'] = self.get_ipython
1369 1369
1370 1370 ns['exit'] = self.exiter
1371 1371 ns['quit'] = self.exiter
1372 1372
1373 1373 # Sync what we've added so far to user_ns_hidden so these aren't seen
1374 1374 # by %who
1375 1375 self.user_ns_hidden.update(ns)
1376 1376
1377 1377 # Anything put into ns now would show up in %who. Think twice before
1378 1378 # putting anything here, as we really want %who to show the user their
1379 1379 # stuff, not our variables.
1380 1380
1381 1381 # Finally, update the real user's namespace
1382 1382 self.user_ns.update(ns)
1383 1383
1384 1384 @property
1385 1385 def all_ns_refs(self):
1386 1386 """Get a list of references to all the namespace dictionaries in which
1387 1387 IPython might store a user-created object.
1388 1388
1389 1389 Note that this does not include the displayhook, which also caches
1390 1390 objects from the output."""
1391 1391 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1392 1392 [m.__dict__ for m in self._main_mod_cache.values()]
1393 1393
1394 1394 def reset(self, new_session=True):
1395 1395 """Clear all internal namespaces, and attempt to release references to
1396 1396 user objects.
1397 1397
1398 1398 If new_session is True, a new history session will be opened.
1399 1399 """
1400 1400 # Clear histories
1401 1401 self.history_manager.reset(new_session)
1402 1402 # Reset counter used to index all histories
1403 1403 if new_session:
1404 1404 self.execution_count = 1
1405 1405
1406 1406 # Reset last execution result
1407 1407 self.last_execution_succeeded = True
1408 1408 self.last_execution_result = None
1409 1409
1410 1410 # Flush cached output items
1411 1411 if self.displayhook.do_full_cache:
1412 1412 self.displayhook.flush()
1413 1413
1414 1414 # The main execution namespaces must be cleared very carefully,
1415 1415 # skipping the deletion of the builtin-related keys, because doing so
1416 1416 # would cause errors in many object's __del__ methods.
1417 1417 if self.user_ns is not self.user_global_ns:
1418 1418 self.user_ns.clear()
1419 1419 ns = self.user_global_ns
1420 1420 drop_keys = set(ns.keys())
1421 1421 drop_keys.discard('__builtin__')
1422 1422 drop_keys.discard('__builtins__')
1423 1423 drop_keys.discard('__name__')
1424 1424 for k in drop_keys:
1425 1425 del ns[k]
1426 1426
1427 1427 self.user_ns_hidden.clear()
1428 1428
1429 1429 # Restore the user namespaces to minimal usability
1430 1430 self.init_user_ns()
1431 1431
1432 1432 # Restore the default and user aliases
1433 1433 self.alias_manager.clear_aliases()
1434 1434 self.alias_manager.init_aliases()
1435 1435
1436 1436 # Flush the private list of module references kept for script
1437 1437 # execution protection
1438 1438 self.clear_main_mod_cache()
1439 1439
1440 1440 def del_var(self, varname, by_name=False):
1441 1441 """Delete a variable from the various namespaces, so that, as
1442 1442 far as possible, we're not keeping any hidden references to it.
1443 1443
1444 1444 Parameters
1445 1445 ----------
1446 1446 varname : str
1447 1447 The name of the variable to delete.
1448 1448 by_name : bool
1449 1449 If True, delete variables with the given name in each
1450 1450 namespace. If False (default), find the variable in the user
1451 1451 namespace, and delete references to it.
1452 1452 """
1453 1453 if varname in ('__builtin__', '__builtins__'):
1454 1454 raise ValueError("Refusing to delete %s" % varname)
1455 1455
1456 1456 ns_refs = self.all_ns_refs
1457 1457
1458 1458 if by_name: # Delete by name
1459 1459 for ns in ns_refs:
1460 1460 try:
1461 1461 del ns[varname]
1462 1462 except KeyError:
1463 1463 pass
1464 1464 else: # Delete by object
1465 1465 try:
1466 1466 obj = self.user_ns[varname]
1467 1467 except KeyError:
1468 1468 raise NameError("name '%s' is not defined" % varname)
1469 1469 # Also check in output history
1470 1470 ns_refs.append(self.history_manager.output_hist)
1471 1471 for ns in ns_refs:
1472 1472 to_delete = [n for n, o in ns.items() if o is obj]
1473 1473 for name in to_delete:
1474 1474 del ns[name]
1475 1475
1476 1476 # Ensure it is removed from the last execution result
1477 1477 if self.last_execution_result.result is obj:
1478 1478 self.last_execution_result = None
1479 1479
1480 1480 # displayhook keeps extra references, but not in a dictionary
1481 1481 for name in ('_', '__', '___'):
1482 1482 if getattr(self.displayhook, name) is obj:
1483 1483 setattr(self.displayhook, name, None)
1484 1484
1485 1485 def reset_selective(self, regex=None):
1486 1486 """Clear selective variables from internal namespaces based on a
1487 1487 specified regular expression.
1488 1488
1489 1489 Parameters
1490 1490 ----------
1491 1491 regex : string or compiled pattern, optional
1492 1492 A regular expression pattern that will be used in searching
1493 1493 variable names in the users namespaces.
1494 1494 """
1495 1495 if regex is not None:
1496 1496 try:
1497 1497 m = re.compile(regex)
1498 1498 except TypeError:
1499 1499 raise TypeError('regex must be a string or compiled pattern')
1500 1500 # Search for keys in each namespace that match the given regex
1501 1501 # If a match is found, delete the key/value pair.
1502 1502 for ns in self.all_ns_refs:
1503 1503 for var in ns:
1504 1504 if m.search(var):
1505 1505 del ns[var]
1506 1506
1507 1507 def push(self, variables, interactive=True):
1508 1508 """Inject a group of variables into the IPython user namespace.
1509 1509
1510 1510 Parameters
1511 1511 ----------
1512 1512 variables : dict, str or list/tuple of str
1513 1513 The variables to inject into the user's namespace. If a dict, a
1514 1514 simple update is done. If a str, the string is assumed to have
1515 1515 variable names separated by spaces. A list/tuple of str can also
1516 1516 be used to give the variable names. If just the variable names are
1517 1517 give (list/tuple/str) then the variable values looked up in the
1518 1518 callers frame.
1519 1519 interactive : bool
1520 1520 If True (default), the variables will be listed with the ``who``
1521 1521 magic.
1522 1522 """
1523 1523 vdict = None
1524 1524
1525 1525 # We need a dict of name/value pairs to do namespace updates.
1526 1526 if isinstance(variables, dict):
1527 1527 vdict = variables
1528 1528 elif isinstance(variables, (str, list, tuple)):
1529 1529 if isinstance(variables, str):
1530 1530 vlist = variables.split()
1531 1531 else:
1532 1532 vlist = variables
1533 1533 vdict = {}
1534 1534 cf = sys._getframe(1)
1535 1535 for name in vlist:
1536 1536 try:
1537 1537 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1538 1538 except:
1539 1539 print('Could not get variable %s from %s' %
1540 1540 (name,cf.f_code.co_name))
1541 1541 else:
1542 1542 raise ValueError('variables must be a dict/str/list/tuple')
1543 1543
1544 1544 # Propagate variables to user namespace
1545 1545 self.user_ns.update(vdict)
1546 1546
1547 1547 # And configure interactive visibility
1548 1548 user_ns_hidden = self.user_ns_hidden
1549 1549 if interactive:
1550 1550 for name in vdict:
1551 1551 user_ns_hidden.pop(name, None)
1552 1552 else:
1553 1553 user_ns_hidden.update(vdict)
1554 1554
1555 1555 def drop_by_id(self, variables):
1556 1556 """Remove a dict of variables from the user namespace, if they are the
1557 1557 same as the values in the dictionary.
1558 1558
1559 1559 This is intended for use by extensions: variables that they've added can
1560 1560 be taken back out if they are unloaded, without removing any that the
1561 1561 user has overwritten.
1562 1562
1563 1563 Parameters
1564 1564 ----------
1565 1565 variables : dict
1566 1566 A dictionary mapping object names (as strings) to the objects.
1567 1567 """
1568 1568 for name, obj in variables.items():
1569 1569 if name in self.user_ns and self.user_ns[name] is obj:
1570 1570 del self.user_ns[name]
1571 1571 self.user_ns_hidden.pop(name, None)
1572 1572
1573 1573 #-------------------------------------------------------------------------
1574 1574 # Things related to object introspection
1575 1575 #-------------------------------------------------------------------------
1576 1576
1577 1577 def _ofind(self, oname, namespaces=None):
1578 1578 """Find an object in the available namespaces.
1579 1579
1580 1580 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1581 1581
1582 1582 Has special code to detect magic functions.
1583 1583 """
1584 1584 oname = oname.strip()
1585 1585 if not oname.startswith(ESC_MAGIC) and \
1586 1586 not oname.startswith(ESC_MAGIC2) and \
1587 1587 not all(a.isidentifier() for a in oname.split(".")):
1588 1588 return {'found': False}
1589 1589
1590 1590 if namespaces is None:
1591 1591 # Namespaces to search in:
1592 1592 # Put them in a list. The order is important so that we
1593 1593 # find things in the same order that Python finds them.
1594 1594 namespaces = [ ('Interactive', self.user_ns),
1595 1595 ('Interactive (global)', self.user_global_ns),
1596 1596 ('Python builtin', builtin_mod.__dict__),
1597 1597 ]
1598 1598
1599 1599 ismagic = False
1600 1600 isalias = False
1601 1601 found = False
1602 1602 ospace = None
1603 1603 parent = None
1604 1604 obj = None
1605 1605
1606 1606
1607 1607 # Look for the given name by splitting it in parts. If the head is
1608 1608 # found, then we look for all the remaining parts as members, and only
1609 1609 # declare success if we can find them all.
1610 1610 oname_parts = oname.split('.')
1611 1611 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1612 1612 for nsname,ns in namespaces:
1613 1613 try:
1614 1614 obj = ns[oname_head]
1615 1615 except KeyError:
1616 1616 continue
1617 1617 else:
1618 1618 for idx, part in enumerate(oname_rest):
1619 1619 try:
1620 1620 parent = obj
1621 1621 # The last part is looked up in a special way to avoid
1622 1622 # descriptor invocation as it may raise or have side
1623 1623 # effects.
1624 1624 if idx == len(oname_rest) - 1:
1625 1625 obj = self._getattr_property(obj, part)
1626 1626 else:
1627 1627 obj = getattr(obj, part)
1628 1628 except:
1629 1629 # Blanket except b/c some badly implemented objects
1630 1630 # allow __getattr__ to raise exceptions other than
1631 1631 # AttributeError, which then crashes IPython.
1632 1632 break
1633 1633 else:
1634 1634 # If we finish the for loop (no break), we got all members
1635 1635 found = True
1636 1636 ospace = nsname
1637 1637 break # namespace loop
1638 1638
1639 1639 # Try to see if it's magic
1640 1640 if not found:
1641 1641 obj = None
1642 1642 if oname.startswith(ESC_MAGIC2):
1643 1643 oname = oname.lstrip(ESC_MAGIC2)
1644 1644 obj = self.find_cell_magic(oname)
1645 1645 elif oname.startswith(ESC_MAGIC):
1646 1646 oname = oname.lstrip(ESC_MAGIC)
1647 1647 obj = self.find_line_magic(oname)
1648 1648 else:
1649 1649 # search without prefix, so run? will find %run?
1650 1650 obj = self.find_line_magic(oname)
1651 1651 if obj is None:
1652 1652 obj = self.find_cell_magic(oname)
1653 1653 if obj is not None:
1654 1654 found = True
1655 1655 ospace = 'IPython internal'
1656 1656 ismagic = True
1657 1657 isalias = isinstance(obj, Alias)
1658 1658
1659 1659 # Last try: special-case some literals like '', [], {}, etc:
1660 1660 if not found and oname_head in ["''",'""','[]','{}','()']:
1661 1661 obj = eval(oname_head)
1662 1662 found = True
1663 1663 ospace = 'Interactive'
1664 1664
1665 1665 return {
1666 1666 'obj':obj,
1667 1667 'found':found,
1668 1668 'parent':parent,
1669 1669 'ismagic':ismagic,
1670 1670 'isalias':isalias,
1671 1671 'namespace':ospace
1672 1672 }
1673 1673
1674 1674 @staticmethod
1675 1675 def _getattr_property(obj, attrname):
1676 1676 """Property-aware getattr to use in object finding.
1677 1677
1678 1678 If attrname represents a property, return it unevaluated (in case it has
1679 1679 side effects or raises an error.
1680 1680
1681 1681 """
1682 1682 if not isinstance(obj, type):
1683 1683 try:
1684 1684 # `getattr(type(obj), attrname)` is not guaranteed to return
1685 1685 # `obj`, but does so for property:
1686 1686 #
1687 1687 # property.__get__(self, None, cls) -> self
1688 1688 #
1689 1689 # The universal alternative is to traverse the mro manually
1690 1690 # searching for attrname in class dicts.
1691 1691 attr = getattr(type(obj), attrname)
1692 1692 except AttributeError:
1693 1693 pass
1694 1694 else:
1695 1695 # This relies on the fact that data descriptors (with both
1696 1696 # __get__ & __set__ magic methods) take precedence over
1697 1697 # instance-level attributes:
1698 1698 #
1699 1699 # class A(object):
1700 1700 # @property
1701 1701 # def foobar(self): return 123
1702 1702 # a = A()
1703 1703 # a.__dict__['foobar'] = 345
1704 1704 # a.foobar # == 123
1705 1705 #
1706 1706 # So, a property may be returned right away.
1707 1707 if isinstance(attr, property):
1708 1708 return attr
1709 1709
1710 1710 # Nothing helped, fall back.
1711 1711 return getattr(obj, attrname)
1712 1712
1713 1713 def _object_find(self, oname, namespaces=None):
1714 1714 """Find an object and return a struct with info about it."""
1715 1715 return Struct(self._ofind(oname, namespaces))
1716 1716
1717 1717 def _inspect(self, meth, oname, namespaces=None, **kw):
1718 1718 """Generic interface to the inspector system.
1719 1719
1720 1720 This function is meant to be called by pdef, pdoc & friends.
1721 1721 """
1722 1722 info = self._object_find(oname, namespaces)
1723 1723 docformat = sphinxify if self.sphinxify_docstring else None
1724 1724 if info.found:
1725 1725 pmethod = getattr(self.inspector, meth)
1726 1726 # TODO: only apply format_screen to the plain/text repr of the mime
1727 1727 # bundle.
1728 1728 formatter = format_screen if info.ismagic else docformat
1729 1729 if meth == 'pdoc':
1730 1730 pmethod(info.obj, oname, formatter)
1731 1731 elif meth == 'pinfo':
1732 1732 pmethod(info.obj, oname, formatter, info,
1733 1733 enable_html_pager=self.enable_html_pager, **kw)
1734 1734 else:
1735 1735 pmethod(info.obj, oname)
1736 1736 else:
1737 1737 print('Object `%s` not found.' % oname)
1738 1738 return 'not found' # so callers can take other action
1739 1739
1740 1740 def object_inspect(self, oname, detail_level=0):
1741 1741 """Get object info about oname"""
1742 1742 with self.builtin_trap:
1743 1743 info = self._object_find(oname)
1744 1744 if info.found:
1745 1745 return self.inspector.info(info.obj, oname, info=info,
1746 1746 detail_level=detail_level
1747 1747 )
1748 1748 else:
1749 1749 return oinspect.object_info(name=oname, found=False)
1750 1750
1751 1751 def object_inspect_text(self, oname, detail_level=0):
1752 1752 """Get object info as formatted text"""
1753 1753 return self.object_inspect_mime(oname, detail_level)['text/plain']
1754 1754
1755 1755 def object_inspect_mime(self, oname, detail_level=0):
1756 1756 """Get object info as a mimebundle of formatted representations.
1757 1757
1758 1758 A mimebundle is a dictionary, keyed by mime-type.
1759 1759 It must always have the key `'text/plain'`.
1760 1760 """
1761 1761 with self.builtin_trap:
1762 1762 info = self._object_find(oname)
1763 1763 if info.found:
1764 1764 return self.inspector._get_info(info.obj, oname, info=info,
1765 1765 detail_level=detail_level
1766 1766 )
1767 1767 else:
1768 1768 raise KeyError(oname)
1769 1769
1770 1770 #-------------------------------------------------------------------------
1771 1771 # Things related to history management
1772 1772 #-------------------------------------------------------------------------
1773 1773
1774 1774 def init_history(self):
1775 1775 """Sets up the command history, and starts regular autosaves."""
1776 1776 self.history_manager = HistoryManager(shell=self, parent=self)
1777 1777 self.configurables.append(self.history_manager)
1778 1778
1779 1779 #-------------------------------------------------------------------------
1780 1780 # Things related to exception handling and tracebacks (not debugging)
1781 1781 #-------------------------------------------------------------------------
1782 1782
1783 1783 debugger_cls = Pdb
1784 1784
1785 1785 def init_traceback_handlers(self, custom_exceptions):
1786 1786 # Syntax error handler.
1787 1787 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
1788 1788
1789 1789 # The interactive one is initialized with an offset, meaning we always
1790 1790 # want to remove the topmost item in the traceback, which is our own
1791 1791 # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
1792 1792 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1793 1793 color_scheme='NoColor',
1794 1794 tb_offset = 1,
1795 1795 check_cache=check_linecache_ipython,
1796 1796 debugger_cls=self.debugger_cls, parent=self)
1797 1797
1798 1798 # The instance will store a pointer to the system-wide exception hook,
1799 1799 # so that runtime code (such as magics) can access it. This is because
1800 1800 # during the read-eval loop, it may get temporarily overwritten.
1801 1801 self.sys_excepthook = sys.excepthook
1802 1802
1803 1803 # and add any custom exception handlers the user may have specified
1804 1804 self.set_custom_exc(*custom_exceptions)
1805 1805
1806 1806 # Set the exception mode
1807 1807 self.InteractiveTB.set_mode(mode=self.xmode)
1808 1808
1809 1809 def set_custom_exc(self, exc_tuple, handler):
1810 1810 """set_custom_exc(exc_tuple, handler)
1811 1811
1812 1812 Set a custom exception handler, which will be called if any of the
1813 1813 exceptions in exc_tuple occur in the mainloop (specifically, in the
1814 1814 run_code() method).
1815 1815
1816 1816 Parameters
1817 1817 ----------
1818 1818
1819 1819 exc_tuple : tuple of exception classes
1820 1820 A *tuple* of exception classes, for which to call the defined
1821 1821 handler. It is very important that you use a tuple, and NOT A
1822 1822 LIST here, because of the way Python's except statement works. If
1823 1823 you only want to trap a single exception, use a singleton tuple::
1824 1824
1825 1825 exc_tuple == (MyCustomException,)
1826 1826
1827 1827 handler : callable
1828 1828 handler must have the following signature::
1829 1829
1830 1830 def my_handler(self, etype, value, tb, tb_offset=None):
1831 1831 ...
1832 1832 return structured_traceback
1833 1833
1834 1834 Your handler must return a structured traceback (a list of strings),
1835 1835 or None.
1836 1836
1837 1837 This will be made into an instance method (via types.MethodType)
1838 1838 of IPython itself, and it will be called if any of the exceptions
1839 1839 listed in the exc_tuple are caught. If the handler is None, an
1840 1840 internal basic one is used, which just prints basic info.
1841 1841
1842 1842 To protect IPython from crashes, if your handler ever raises an
1843 1843 exception or returns an invalid result, it will be immediately
1844 1844 disabled.
1845 1845
1846 1846 WARNING: by putting in your own exception handler into IPython's main
1847 1847 execution loop, you run a very good chance of nasty crashes. This
1848 1848 facility should only be used if you really know what you are doing."""
1849 1849 if not isinstance(exc_tuple, tuple):
1850 1850 raise TypeError("The custom exceptions must be given as a tuple.")
1851 1851
1852 1852 def dummy_handler(self, etype, value, tb, tb_offset=None):
1853 1853 print('*** Simple custom exception handler ***')
1854 1854 print('Exception type :', etype)
1855 1855 print('Exception value:', value)
1856 1856 print('Traceback :', tb)
1857 1857
1858 1858 def validate_stb(stb):
1859 1859 """validate structured traceback return type
1860 1860
1861 1861 return type of CustomTB *should* be a list of strings, but allow
1862 1862 single strings or None, which are harmless.
1863 1863
1864 1864 This function will *always* return a list of strings,
1865 1865 and will raise a TypeError if stb is inappropriate.
1866 1866 """
1867 1867 msg = "CustomTB must return list of strings, not %r" % stb
1868 1868 if stb is None:
1869 1869 return []
1870 1870 elif isinstance(stb, str):
1871 1871 return [stb]
1872 1872 elif not isinstance(stb, list):
1873 1873 raise TypeError(msg)
1874 1874 # it's a list
1875 1875 for line in stb:
1876 1876 # check every element
1877 1877 if not isinstance(line, str):
1878 1878 raise TypeError(msg)
1879 1879 return stb
1880 1880
1881 1881 if handler is None:
1882 1882 wrapped = dummy_handler
1883 1883 else:
1884 1884 def wrapped(self,etype,value,tb,tb_offset=None):
1885 1885 """wrap CustomTB handler, to protect IPython from user code
1886 1886
1887 1887 This makes it harder (but not impossible) for custom exception
1888 1888 handlers to crash IPython.
1889 1889 """
1890 1890 try:
1891 1891 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1892 1892 return validate_stb(stb)
1893 1893 except:
1894 1894 # clear custom handler immediately
1895 1895 self.set_custom_exc((), None)
1896 1896 print("Custom TB Handler failed, unregistering", file=sys.stderr)
1897 1897 # show the exception in handler first
1898 1898 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1899 1899 print(self.InteractiveTB.stb2text(stb))
1900 1900 print("The original exception:")
1901 1901 stb = self.InteractiveTB.structured_traceback(
1902 1902 (etype,value,tb), tb_offset=tb_offset
1903 1903 )
1904 1904 return stb
1905 1905
1906 1906 self.CustomTB = types.MethodType(wrapped,self)
1907 1907 self.custom_exceptions = exc_tuple
1908 1908
1909 1909 def excepthook(self, etype, value, tb):
1910 1910 """One more defense for GUI apps that call sys.excepthook.
1911 1911
1912 1912 GUI frameworks like wxPython trap exceptions and call
1913 1913 sys.excepthook themselves. I guess this is a feature that
1914 1914 enables them to keep running after exceptions that would
1915 1915 otherwise kill their mainloop. This is a bother for IPython
1916 1916 which excepts to catch all of the program exceptions with a try:
1917 1917 except: statement.
1918 1918
1919 1919 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1920 1920 any app directly invokes sys.excepthook, it will look to the user like
1921 1921 IPython crashed. In order to work around this, we can disable the
1922 1922 CrashHandler and replace it with this excepthook instead, which prints a
1923 1923 regular traceback using our InteractiveTB. In this fashion, apps which
1924 1924 call sys.excepthook will generate a regular-looking exception from
1925 1925 IPython, and the CrashHandler will only be triggered by real IPython
1926 1926 crashes.
1927 1927
1928 1928 This hook should be used sparingly, only in places which are not likely
1929 1929 to be true IPython errors.
1930 1930 """
1931 1931 self.showtraceback((etype, value, tb), tb_offset=0)
1932 1932
1933 1933 def _get_exc_info(self, exc_tuple=None):
1934 1934 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1935 1935
1936 1936 Ensures sys.last_type,value,traceback hold the exc_info we found,
1937 1937 from whichever source.
1938 1938
1939 1939 raises ValueError if none of these contain any information
1940 1940 """
1941 1941 if exc_tuple is None:
1942 1942 etype, value, tb = sys.exc_info()
1943 1943 else:
1944 1944 etype, value, tb = exc_tuple
1945 1945
1946 1946 if etype is None:
1947 1947 if hasattr(sys, 'last_type'):
1948 1948 etype, value, tb = sys.last_type, sys.last_value, \
1949 1949 sys.last_traceback
1950 1950
1951 1951 if etype is None:
1952 1952 raise ValueError("No exception to find")
1953 1953
1954 1954 # Now store the exception info in sys.last_type etc.
1955 1955 # WARNING: these variables are somewhat deprecated and not
1956 1956 # necessarily safe to use in a threaded environment, but tools
1957 1957 # like pdb depend on their existence, so let's set them. If we
1958 1958 # find problems in the field, we'll need to revisit their use.
1959 1959 sys.last_type = etype
1960 1960 sys.last_value = value
1961 1961 sys.last_traceback = tb
1962 1962
1963 1963 return etype, value, tb
1964 1964
1965 1965 def show_usage_error(self, exc):
1966 1966 """Show a short message for UsageErrors
1967 1967
1968 1968 These are special exceptions that shouldn't show a traceback.
1969 1969 """
1970 1970 print("UsageError: %s" % exc, file=sys.stderr)
1971 1971
1972 1972 def get_exception_only(self, exc_tuple=None):
1973 1973 """
1974 1974 Return as a string (ending with a newline) the exception that
1975 1975 just occurred, without any traceback.
1976 1976 """
1977 1977 etype, value, tb = self._get_exc_info(exc_tuple)
1978 1978 msg = traceback.format_exception_only(etype, value)
1979 1979 return ''.join(msg)
1980 1980
1981 1981 def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
1982 1982 exception_only=False, running_compiled_code=False):
1983 1983 """Display the exception that just occurred.
1984 1984
1985 1985 If nothing is known about the exception, this is the method which
1986 1986 should be used throughout the code for presenting user tracebacks,
1987 1987 rather than directly invoking the InteractiveTB object.
1988 1988
1989 1989 A specific showsyntaxerror() also exists, but this method can take
1990 1990 care of calling it if needed, so unless you are explicitly catching a
1991 1991 SyntaxError exception, don't try to analyze the stack manually and
1992 1992 simply call this method."""
1993 1993
1994 1994 try:
1995 1995 try:
1996 1996 etype, value, tb = self._get_exc_info(exc_tuple)
1997 1997 except ValueError:
1998 1998 print('No traceback available to show.', file=sys.stderr)
1999 1999 return
2000 2000
2001 2001 if issubclass(etype, SyntaxError):
2002 2002 # Though this won't be called by syntax errors in the input
2003 2003 # line, there may be SyntaxError cases with imported code.
2004 2004 self.showsyntaxerror(filename, running_compiled_code)
2005 2005 elif etype is UsageError:
2006 2006 self.show_usage_error(value)
2007 2007 else:
2008 2008 if exception_only:
2009 2009 stb = ['An exception has occurred, use %tb to see '
2010 2010 'the full traceback.\n']
2011 2011 stb.extend(self.InteractiveTB.get_exception_only(etype,
2012 2012 value))
2013 2013 else:
2014 2014 try:
2015 2015 # Exception classes can customise their traceback - we
2016 2016 # use this in IPython.parallel for exceptions occurring
2017 2017 # in the engines. This should return a list of strings.
2018 2018 stb = value._render_traceback_()
2019 2019 except Exception:
2020 2020 stb = self.InteractiveTB.structured_traceback(etype,
2021 2021 value, tb, tb_offset=tb_offset)
2022 2022
2023 2023 self._showtraceback(etype, value, stb)
2024 2024 if self.call_pdb:
2025 2025 # drop into debugger
2026 2026 self.debugger(force=True)
2027 2027 return
2028 2028
2029 2029 # Actually show the traceback
2030 2030 self._showtraceback(etype, value, stb)
2031 2031
2032 2032 except KeyboardInterrupt:
2033 2033 print('\n' + self.get_exception_only(), file=sys.stderr)
2034 2034
2035 2035 def _showtraceback(self, etype, evalue, stb):
2036 2036 """Actually show a traceback.
2037 2037
2038 2038 Subclasses may override this method to put the traceback on a different
2039 2039 place, like a side channel.
2040 2040 """
2041 2041 print(self.InteractiveTB.stb2text(stb))
2042 2042
2043 2043 def showsyntaxerror(self, filename=None, running_compiled_code=False):
2044 2044 """Display the syntax error that just occurred.
2045 2045
2046 2046 This doesn't display a stack trace because there isn't one.
2047 2047
2048 2048 If a filename is given, it is stuffed in the exception instead
2049 2049 of what was there before (because Python's parser always uses
2050 2050 "<string>" when reading from a string).
2051 2051
2052 2052 If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
2053 2053 longer stack trace will be displayed.
2054 2054 """
2055 2055 etype, value, last_traceback = self._get_exc_info()
2056 2056
2057 2057 if filename and issubclass(etype, SyntaxError):
2058 2058 try:
2059 2059 value.filename = filename
2060 2060 except:
2061 2061 # Not the format we expect; leave it alone
2062 2062 pass
2063 2063
2064 2064 # If the error occurred when executing compiled code, we should provide full stacktrace.
2065 2065 elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
2066 2066 stb = self.SyntaxTB.structured_traceback(etype, value, elist)
2067 2067 self._showtraceback(etype, value, stb)
2068 2068
2069 2069 # This is overridden in TerminalInteractiveShell to show a message about
2070 2070 # the %paste magic.
2071 2071 def showindentationerror(self):
2072 2072 """Called by _run_cell when there's an IndentationError in code entered
2073 2073 at the prompt.
2074 2074
2075 2075 This is overridden in TerminalInteractiveShell to show a message about
2076 2076 the %paste magic."""
2077 2077 self.showsyntaxerror()
2078 2078
2079 2079 #-------------------------------------------------------------------------
2080 2080 # Things related to readline
2081 2081 #-------------------------------------------------------------------------
2082 2082
2083 2083 def init_readline(self):
2084 2084 """DEPRECATED
2085 2085
2086 2086 Moved to terminal subclass, here only to simplify the init logic."""
2087 2087 # Set a number of methods that depend on readline to be no-op
2088 2088 warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
2089 2089 DeprecationWarning, stacklevel=2)
2090 2090 self.set_custom_completer = no_op
2091 2091
2092 2092 @skip_doctest
2093 2093 def set_next_input(self, s, replace=False):
2094 2094 """ Sets the 'default' input string for the next command line.
2095 2095
2096 2096 Example::
2097 2097
2098 2098 In [1]: _ip.set_next_input("Hello Word")
2099 2099 In [2]: Hello Word_ # cursor is here
2100 2100 """
2101 2101 self.rl_next_input = s
2102 2102
2103 2103 def _indent_current_str(self):
2104 2104 """return the current level of indentation as a string"""
2105 2105 return self.input_splitter.get_indent_spaces() * ' '
2106 2106
2107 2107 #-------------------------------------------------------------------------
2108 2108 # Things related to text completion
2109 2109 #-------------------------------------------------------------------------
2110 2110
2111 2111 def init_completer(self):
2112 2112 """Initialize the completion machinery.
2113 2113
2114 2114 This creates completion machinery that can be used by client code,
2115 2115 either interactively in-process (typically triggered by the readline
2116 2116 library), programmatically (such as in test suites) or out-of-process
2117 2117 (typically over the network by remote frontends).
2118 2118 """
2119 2119 from IPython.core.completer import IPCompleter
2120 2120 from IPython.core.completerlib import (module_completer,
2121 2121 magic_run_completer, cd_completer, reset_completer)
2122 2122
2123 2123 self.Completer = IPCompleter(shell=self,
2124 2124 namespace=self.user_ns,
2125 2125 global_namespace=self.user_global_ns,
2126 2126 parent=self,
2127 2127 )
2128 2128 self.configurables.append(self.Completer)
2129 2129
2130 2130 # Add custom completers to the basic ones built into IPCompleter
2131 2131 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
2132 2132 self.strdispatchers['complete_command'] = sdisp
2133 2133 self.Completer.custom_completers = sdisp
2134 2134
2135 2135 self.set_hook('complete_command', module_completer, str_key = 'import')
2136 2136 self.set_hook('complete_command', module_completer, str_key = 'from')
2137 2137 self.set_hook('complete_command', module_completer, str_key = '%aimport')
2138 2138 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
2139 2139 self.set_hook('complete_command', cd_completer, str_key = '%cd')
2140 2140 self.set_hook('complete_command', reset_completer, str_key = '%reset')
2141 2141
2142 2142 @skip_doctest
2143 2143 def complete(self, text, line=None, cursor_pos=None):
2144 2144 """Return the completed text and a list of completions.
2145 2145
2146 2146 Parameters
2147 2147 ----------
2148 2148
2149 2149 text : string
2150 2150 A string of text to be completed on. It can be given as empty and
2151 2151 instead a line/position pair are given. In this case, the
2152 2152 completer itself will split the line like readline does.
2153 2153
2154 2154 line : string, optional
2155 2155 The complete line that text is part of.
2156 2156
2157 2157 cursor_pos : int, optional
2158 2158 The position of the cursor on the input line.
2159 2159
2160 2160 Returns
2161 2161 -------
2162 2162 text : string
2163 2163 The actual text that was completed.
2164 2164
2165 2165 matches : list
2166 2166 A sorted list with all possible completions.
2167 2167
2168 2168 The optional arguments allow the completion to take more context into
2169 2169 account, and are part of the low-level completion API.
2170 2170
2171 2171 This is a wrapper around the completion mechanism, similar to what
2172 2172 readline does at the command line when the TAB key is hit. By
2173 2173 exposing it as a method, it can be used by other non-readline
2174 2174 environments (such as GUIs) for text completion.
2175 2175
2176 2176 Simple usage example:
2177 2177
2178 2178 In [1]: x = 'hello'
2179 2179
2180 2180 In [2]: _ip.complete('x.l')
2181 2181 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
2182 2182 """
2183 2183
2184 2184 # Inject names into __builtin__ so we can complete on the added names.
2185 2185 with self.builtin_trap:
2186 2186 return self.Completer.complete(text, line, cursor_pos)
2187 2187
2188 2188 def set_custom_completer(self, completer, pos=0):
2189 2189 """Adds a new custom completer function.
2190 2190
2191 2191 The position argument (defaults to 0) is the index in the completers
2192 2192 list where you want the completer to be inserted."""
2193 2193
2194 2194 newcomp = types.MethodType(completer,self.Completer)
2195 2195 self.Completer.matchers.insert(pos,newcomp)
2196 2196
2197 2197 def set_completer_frame(self, frame=None):
2198 2198 """Set the frame of the completer."""
2199 2199 if frame:
2200 2200 self.Completer.namespace = frame.f_locals
2201 2201 self.Completer.global_namespace = frame.f_globals
2202 2202 else:
2203 2203 self.Completer.namespace = self.user_ns
2204 2204 self.Completer.global_namespace = self.user_global_ns
2205 2205
2206 2206 #-------------------------------------------------------------------------
2207 2207 # Things related to magics
2208 2208 #-------------------------------------------------------------------------
2209 2209
2210 2210 def init_magics(self):
2211 2211 from IPython.core import magics as m
2212 2212 self.magics_manager = magic.MagicsManager(shell=self,
2213 2213 parent=self,
2214 2214 user_magics=m.UserMagics(self))
2215 2215 self.configurables.append(self.magics_manager)
2216 2216
2217 2217 # Expose as public API from the magics manager
2218 2218 self.register_magics = self.magics_manager.register
2219 2219
2220 2220 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2221 2221 m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
2222 2222 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2223 2223 m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
2224 2224 m.PylabMagics, m.ScriptMagics,
2225 2225 )
2226 2226 if sys.version_info >(3,5):
2227 2227 self.register_magics(m.AsyncMagics)
2228 2228
2229 2229 # Register Magic Aliases
2230 2230 mman = self.magics_manager
2231 2231 # FIXME: magic aliases should be defined by the Magics classes
2232 2232 # or in MagicsManager, not here
2233 2233 mman.register_alias('ed', 'edit')
2234 2234 mman.register_alias('hist', 'history')
2235 2235 mman.register_alias('rep', 'recall')
2236 2236 mman.register_alias('SVG', 'svg', 'cell')
2237 2237 mman.register_alias('HTML', 'html', 'cell')
2238 2238 mman.register_alias('file', 'writefile', 'cell')
2239 2239
2240 2240 # FIXME: Move the color initialization to the DisplayHook, which
2241 2241 # should be split into a prompt manager and displayhook. We probably
2242 2242 # even need a centralize colors management object.
2243 2243 self.run_line_magic('colors', self.colors)
2244 2244
2245 2245 # Defined here so that it's included in the documentation
2246 2246 @functools.wraps(magic.MagicsManager.register_function)
2247 2247 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2248 2248 self.magics_manager.register_function(func,
2249 2249 magic_kind=magic_kind, magic_name=magic_name)
2250 2250
2251 2251 def run_line_magic(self, magic_name, line, _stack_depth=1):
2252 2252 """Execute the given line magic.
2253 2253
2254 2254 Parameters
2255 2255 ----------
2256 2256 magic_name : str
2257 2257 Name of the desired magic function, without '%' prefix.
2258 2258
2259 2259 line : str
2260 2260 The rest of the input line as a single string.
2261 2261
2262 2262 _stack_depth : int
2263 2263 If run_line_magic() is called from magic() then _stack_depth=2.
2264 2264 This is added to ensure backward compatibility for use of 'get_ipython().magic()'
2265 2265 """
2266 2266 fn = self.find_line_magic(magic_name)
2267 2267 if fn is None:
2268 2268 cm = self.find_cell_magic(magic_name)
2269 2269 etpl = "Line magic function `%%%s` not found%s."
2270 2270 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2271 2271 'did you mean that instead?)' % magic_name )
2272 2272 raise UsageError(etpl % (magic_name, extra))
2273 2273 else:
2274 2274 # Note: this is the distance in the stack to the user's frame.
2275 2275 # This will need to be updated if the internal calling logic gets
2276 2276 # refactored, or else we'll be expanding the wrong variables.
2277 2277
2278 2278 # Determine stack_depth depending on where run_line_magic() has been called
2279 2279 stack_depth = _stack_depth
2280 2280 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2281 2281 # magic has opted out of var_expand
2282 2282 magic_arg_s = line
2283 2283 else:
2284 2284 magic_arg_s = self.var_expand(line, stack_depth)
2285 2285 # Put magic args in a list so we can call with f(*a) syntax
2286 2286 args = [magic_arg_s]
2287 2287 kwargs = {}
2288 2288 # Grab local namespace if we need it:
2289 2289 if getattr(fn, "needs_local_scope", False):
2290 2290 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2291 2291 with self.builtin_trap:
2292 2292 result = fn(*args, **kwargs)
2293 2293 return result
2294 2294
2295 2295 def run_cell_magic(self, magic_name, line, cell):
2296 2296 """Execute the given cell magic.
2297 2297
2298 2298 Parameters
2299 2299 ----------
2300 2300 magic_name : str
2301 2301 Name of the desired magic function, without '%' prefix.
2302 2302
2303 2303 line : str
2304 2304 The rest of the first input line as a single string.
2305 2305
2306 2306 cell : str
2307 2307 The body of the cell as a (possibly multiline) string.
2308 2308 """
2309 2309 fn = self.find_cell_magic(magic_name)
2310 2310 if fn is None:
2311 2311 lm = self.find_line_magic(magic_name)
2312 2312 etpl = "Cell magic `%%{0}` not found{1}."
2313 2313 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2314 2314 'did you mean that instead?)'.format(magic_name))
2315 2315 raise UsageError(etpl.format(magic_name, extra))
2316 2316 elif cell == '':
2317 2317 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2318 2318 if self.find_line_magic(magic_name) is not None:
2319 2319 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2320 2320 raise UsageError(message)
2321 2321 else:
2322 2322 # Note: this is the distance in the stack to the user's frame.
2323 2323 # This will need to be updated if the internal calling logic gets
2324 2324 # refactored, or else we'll be expanding the wrong variables.
2325 2325 stack_depth = 2
2326 2326 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2327 2327 # magic has opted out of var_expand
2328 2328 magic_arg_s = line
2329 2329 else:
2330 2330 magic_arg_s = self.var_expand(line, stack_depth)
2331 2331 with self.builtin_trap:
2332 2332 result = fn(magic_arg_s, cell)
2333 2333 return result
2334 2334
2335 2335 def find_line_magic(self, magic_name):
2336 2336 """Find and return a line magic by name.
2337 2337
2338 2338 Returns None if the magic isn't found."""
2339 2339 return self.magics_manager.magics['line'].get(magic_name)
2340 2340
2341 2341 def find_cell_magic(self, magic_name):
2342 2342 """Find and return a cell magic by name.
2343 2343
2344 2344 Returns None if the magic isn't found."""
2345 2345 return self.magics_manager.magics['cell'].get(magic_name)
2346 2346
2347 2347 def find_magic(self, magic_name, magic_kind='line'):
2348 2348 """Find and return a magic of the given type by name.
2349 2349
2350 2350 Returns None if the magic isn't found."""
2351 2351 return self.magics_manager.magics[magic_kind].get(magic_name)
2352 2352
2353 2353 def magic(self, arg_s):
2354 2354 """DEPRECATED. Use run_line_magic() instead.
2355 2355
2356 2356 Call a magic function by name.
2357 2357
2358 2358 Input: a string containing the name of the magic function to call and
2359 2359 any additional arguments to be passed to the magic.
2360 2360
2361 2361 magic('name -opt foo bar') is equivalent to typing at the ipython
2362 2362 prompt:
2363 2363
2364 2364 In[1]: %name -opt foo bar
2365 2365
2366 2366 To call a magic without arguments, simply use magic('name').
2367 2367
2368 2368 This provides a proper Python function to call IPython's magics in any
2369 2369 valid Python code you can type at the interpreter, including loops and
2370 2370 compound statements.
2371 2371 """
2372 2372 # TODO: should we issue a loud deprecation warning here?
2373 2373 magic_name, _, magic_arg_s = arg_s.partition(' ')
2374 2374 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2375 2375 return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
2376 2376
2377 2377 #-------------------------------------------------------------------------
2378 2378 # Things related to macros
2379 2379 #-------------------------------------------------------------------------
2380 2380
2381 2381 def define_macro(self, name, themacro):
2382 2382 """Define a new macro
2383 2383
2384 2384 Parameters
2385 2385 ----------
2386 2386 name : str
2387 2387 The name of the macro.
2388 2388 themacro : str or Macro
2389 2389 The action to do upon invoking the macro. If a string, a new
2390 2390 Macro object is created by passing the string to it.
2391 2391 """
2392 2392
2393 2393 from IPython.core import macro
2394 2394
2395 2395 if isinstance(themacro, str):
2396 2396 themacro = macro.Macro(themacro)
2397 2397 if not isinstance(themacro, macro.Macro):
2398 2398 raise ValueError('A macro must be a string or a Macro instance.')
2399 2399 self.user_ns[name] = themacro
2400 2400
2401 2401 #-------------------------------------------------------------------------
2402 2402 # Things related to the running of system commands
2403 2403 #-------------------------------------------------------------------------
2404 2404
2405 2405 def system_piped(self, cmd):
2406 2406 """Call the given cmd in a subprocess, piping stdout/err
2407 2407
2408 2408 Parameters
2409 2409 ----------
2410 2410 cmd : str
2411 2411 Command to execute (can not end in '&', as background processes are
2412 2412 not supported. Should not be a command that expects input
2413 2413 other than simple text.
2414 2414 """
2415 2415 if cmd.rstrip().endswith('&'):
2416 2416 # this is *far* from a rigorous test
2417 2417 # We do not support backgrounding processes because we either use
2418 2418 # pexpect or pipes to read from. Users can always just call
2419 2419 # os.system() or use ip.system=ip.system_raw
2420 2420 # if they really want a background process.
2421 2421 raise OSError("Background processes not supported.")
2422 2422
2423 2423 # we explicitly do NOT return the subprocess status code, because
2424 2424 # a non-None value would trigger :func:`sys.displayhook` calls.
2425 2425 # Instead, we store the exit_code in user_ns.
2426 2426 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2427 2427
2428 2428 def system_raw(self, cmd):
2429 2429 """Call the given cmd in a subprocess using os.system on Windows or
2430 2430 subprocess.call using the system shell on other platforms.
2431 2431
2432 2432 Parameters
2433 2433 ----------
2434 2434 cmd : str
2435 2435 Command to execute.
2436 2436 """
2437 2437 cmd = self.var_expand(cmd, depth=1)
2438 2438 # protect os.system from UNC paths on Windows, which it can't handle:
2439 2439 if sys.platform == 'win32':
2440 2440 from IPython.utils._process_win32 import AvoidUNCPath
2441 2441 with AvoidUNCPath() as path:
2442 2442 if path is not None:
2443 2443 cmd = '"pushd %s &&"%s' % (path, cmd)
2444 2444 try:
2445 2445 ec = os.system(cmd)
2446 2446 except KeyboardInterrupt:
2447 2447 print('\n' + self.get_exception_only(), file=sys.stderr)
2448 2448 ec = -2
2449 2449 else:
2450 2450 # For posix the result of the subprocess.call() below is an exit
2451 2451 # code, which by convention is zero for success, positive for
2452 2452 # program failure. Exit codes above 128 are reserved for signals,
2453 2453 # and the formula for converting a signal to an exit code is usually
2454 2454 # signal_number+128. To more easily differentiate between exit
2455 2455 # codes and signals, ipython uses negative numbers. For instance
2456 2456 # since control-c is signal 2 but exit code 130, ipython's
2457 2457 # _exit_code variable will read -2. Note that some shells like
2458 2458 # csh and fish don't follow sh/bash conventions for exit codes.
2459 2459 executable = os.environ.get('SHELL', None)
2460 2460 try:
2461 2461 # Use env shell instead of default /bin/sh
2462 2462 ec = subprocess.call(cmd, shell=True, executable=executable)
2463 2463 except KeyboardInterrupt:
2464 2464 # intercept control-C; a long traceback is not useful here
2465 2465 print('\n' + self.get_exception_only(), file=sys.stderr)
2466 2466 ec = 130
2467 2467 if ec > 128:
2468 2468 ec = -(ec - 128)
2469 2469
2470 2470 # We explicitly do NOT return the subprocess status code, because
2471 2471 # a non-None value would trigger :func:`sys.displayhook` calls.
2472 2472 # Instead, we store the exit_code in user_ns. Note the semantics
2473 2473 # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
2474 2474 # but raising SystemExit(_exit_code) will give status 254!
2475 2475 self.user_ns['_exit_code'] = ec
2476 2476
2477 2477 # use piped system by default, because it is better behaved
2478 2478 system = system_piped
2479 2479
2480 2480 def getoutput(self, cmd, split=True, depth=0):
2481 2481 """Get output (possibly including stderr) from a subprocess.
2482 2482
2483 2483 Parameters
2484 2484 ----------
2485 2485 cmd : str
2486 2486 Command to execute (can not end in '&', as background processes are
2487 2487 not supported.
2488 2488 split : bool, optional
2489 2489 If True, split the output into an IPython SList. Otherwise, an
2490 2490 IPython LSString is returned. These are objects similar to normal
2491 2491 lists and strings, with a few convenience attributes for easier
2492 2492 manipulation of line-based output. You can use '?' on them for
2493 2493 details.
2494 2494 depth : int, optional
2495 2495 How many frames above the caller are the local variables which should
2496 2496 be expanded in the command string? The default (0) assumes that the
2497 2497 expansion variables are in the stack frame calling this function.
2498 2498 """
2499 2499 if cmd.rstrip().endswith('&'):
2500 2500 # this is *far* from a rigorous test
2501 2501 raise OSError("Background processes not supported.")
2502 2502 out = getoutput(self.var_expand(cmd, depth=depth+1))
2503 2503 if split:
2504 2504 out = SList(out.splitlines())
2505 2505 else:
2506 2506 out = LSString(out)
2507 2507 return out
2508 2508
2509 2509 #-------------------------------------------------------------------------
2510 2510 # Things related to aliases
2511 2511 #-------------------------------------------------------------------------
2512 2512
2513 2513 def init_alias(self):
2514 2514 self.alias_manager = AliasManager(shell=self, parent=self)
2515 2515 self.configurables.append(self.alias_manager)
2516 2516
2517 2517 #-------------------------------------------------------------------------
2518 2518 # Things related to extensions
2519 2519 #-------------------------------------------------------------------------
2520 2520
2521 2521 def init_extension_manager(self):
2522 2522 self.extension_manager = ExtensionManager(shell=self, parent=self)
2523 2523 self.configurables.append(self.extension_manager)
2524 2524
2525 2525 #-------------------------------------------------------------------------
2526 2526 # Things related to payloads
2527 2527 #-------------------------------------------------------------------------
2528 2528
2529 2529 def init_payload(self):
2530 2530 self.payload_manager = PayloadManager(parent=self)
2531 2531 self.configurables.append(self.payload_manager)
2532 2532
2533 2533 #-------------------------------------------------------------------------
2534 2534 # Things related to the prefilter
2535 2535 #-------------------------------------------------------------------------
2536 2536
2537 2537 def init_prefilter(self):
2538 2538 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2539 2539 self.configurables.append(self.prefilter_manager)
2540 2540 # Ultimately this will be refactored in the new interpreter code, but
2541 2541 # for now, we should expose the main prefilter method (there's legacy
2542 2542 # code out there that may rely on this).
2543 2543 self.prefilter = self.prefilter_manager.prefilter_lines
2544 2544
2545 2545 def auto_rewrite_input(self, cmd):
2546 2546 """Print to the screen the rewritten form of the user's command.
2547 2547
2548 2548 This shows visual feedback by rewriting input lines that cause
2549 2549 automatic calling to kick in, like::
2550 2550
2551 2551 /f x
2552 2552
2553 2553 into::
2554 2554
2555 2555 ------> f(x)
2556 2556
2557 2557 after the user's input prompt. This helps the user understand that the
2558 2558 input line was transformed automatically by IPython.
2559 2559 """
2560 2560 if not self.show_rewritten_input:
2561 2561 return
2562 2562
2563 2563 # This is overridden in TerminalInteractiveShell to use fancy prompts
2564 2564 print("------> " + cmd)
2565 2565
2566 2566 #-------------------------------------------------------------------------
2567 2567 # Things related to extracting values/expressions from kernel and user_ns
2568 2568 #-------------------------------------------------------------------------
2569 2569
2570 2570 def _user_obj_error(self):
2571 2571 """return simple exception dict
2572 2572
2573 2573 for use in user_expressions
2574 2574 """
2575 2575
2576 2576 etype, evalue, tb = self._get_exc_info()
2577 2577 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2578 2578
2579 2579 exc_info = {
2580 2580 u'status' : 'error',
2581 2581 u'traceback' : stb,
2582 2582 u'ename' : etype.__name__,
2583 2583 u'evalue' : py3compat.safe_unicode(evalue),
2584 2584 }
2585 2585
2586 2586 return exc_info
2587 2587
2588 2588 def _format_user_obj(self, obj):
2589 2589 """format a user object to display dict
2590 2590
2591 2591 for use in user_expressions
2592 2592 """
2593 2593
2594 2594 data, md = self.display_formatter.format(obj)
2595 2595 value = {
2596 2596 'status' : 'ok',
2597 2597 'data' : data,
2598 2598 'metadata' : md,
2599 2599 }
2600 2600 return value
2601 2601
2602 2602 def user_expressions(self, expressions):
2603 2603 """Evaluate a dict of expressions in the user's namespace.
2604 2604
2605 2605 Parameters
2606 2606 ----------
2607 2607 expressions : dict
2608 2608 A dict with string keys and string values. The expression values
2609 2609 should be valid Python expressions, each of which will be evaluated
2610 2610 in the user namespace.
2611 2611
2612 2612 Returns
2613 2613 -------
2614 2614 A dict, keyed like the input expressions dict, with the rich mime-typed
2615 2615 display_data of each value.
2616 2616 """
2617 2617 out = {}
2618 2618 user_ns = self.user_ns
2619 2619 global_ns = self.user_global_ns
2620 2620
2621 2621 for key, expr in expressions.items():
2622 2622 try:
2623 2623 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2624 2624 except:
2625 2625 value = self._user_obj_error()
2626 2626 out[key] = value
2627 2627 return out
2628 2628
2629 2629 #-------------------------------------------------------------------------
2630 2630 # Things related to the running of code
2631 2631 #-------------------------------------------------------------------------
2632 2632
2633 2633 def ex(self, cmd):
2634 2634 """Execute a normal python statement in user namespace."""
2635 2635 with self.builtin_trap:
2636 2636 exec(cmd, self.user_global_ns, self.user_ns)
2637 2637
2638 2638 def ev(self, expr):
2639 2639 """Evaluate python expression expr in user namespace.
2640 2640
2641 2641 Returns the result of evaluation
2642 2642 """
2643 2643 with self.builtin_trap:
2644 2644 return eval(expr, self.user_global_ns, self.user_ns)
2645 2645
2646 2646 def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
2647 2647 """A safe version of the builtin execfile().
2648 2648
2649 2649 This version will never throw an exception, but instead print
2650 2650 helpful error messages to the screen. This only works on pure
2651 2651 Python files with the .py extension.
2652 2652
2653 2653 Parameters
2654 2654 ----------
2655 2655 fname : string
2656 2656 The name of the file to be executed.
2657 2657 where : tuple
2658 2658 One or two namespaces, passed to execfile() as (globals,locals).
2659 2659 If only one is given, it is passed as both.
2660 2660 exit_ignore : bool (False)
2661 2661 If True, then silence SystemExit for non-zero status (it is always
2662 2662 silenced for zero status, as it is so common).
2663 2663 raise_exceptions : bool (False)
2664 2664 If True raise exceptions everywhere. Meant for testing.
2665 2665 shell_futures : bool (False)
2666 2666 If True, the code will share future statements with the interactive
2667 2667 shell. It will both be affected by previous __future__ imports, and
2668 2668 any __future__ imports in the code will affect the shell. If False,
2669 2669 __future__ imports are not shared in either direction.
2670 2670
2671 2671 """
2672 2672 fname = os.path.abspath(os.path.expanduser(fname))
2673 2673
2674 2674 # Make sure we can open the file
2675 2675 try:
2676 2676 with open(fname):
2677 2677 pass
2678 2678 except:
2679 2679 warn('Could not open file <%s> for safe execution.' % fname)
2680 2680 return
2681 2681
2682 2682 # Find things also in current directory. This is needed to mimic the
2683 2683 # behavior of running a script from the system command line, where
2684 2684 # Python inserts the script's directory into sys.path
2685 2685 dname = os.path.dirname(fname)
2686 2686
2687 2687 with prepended_to_syspath(dname), self.builtin_trap:
2688 2688 try:
2689 2689 glob, loc = (where + (None, ))[:2]
2690 2690 py3compat.execfile(
2691 2691 fname, glob, loc,
2692 2692 self.compile if shell_futures else None)
2693 2693 except SystemExit as status:
2694 2694 # If the call was made with 0 or None exit status (sys.exit(0)
2695 2695 # or sys.exit() ), don't bother showing a traceback, as both of
2696 2696 # these are considered normal by the OS:
2697 2697 # > python -c'import sys;sys.exit(0)'; echo $?
2698 2698 # 0
2699 2699 # > python -c'import sys;sys.exit()'; echo $?
2700 2700 # 0
2701 2701 # For other exit status, we show the exception unless
2702 2702 # explicitly silenced, but only in short form.
2703 2703 if status.code:
2704 2704 if raise_exceptions:
2705 2705 raise
2706 2706 if not exit_ignore:
2707 2707 self.showtraceback(exception_only=True)
2708 2708 except:
2709 2709 if raise_exceptions:
2710 2710 raise
2711 2711 # tb offset is 2 because we wrap execfile
2712 2712 self.showtraceback(tb_offset=2)
2713 2713
2714 2714 def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
2715 2715 """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
2716 2716
2717 2717 Parameters
2718 2718 ----------
2719 2719 fname : str
2720 2720 The name of the file to execute. The filename must have a
2721 2721 .ipy or .ipynb extension.
2722 2722 shell_futures : bool (False)
2723 2723 If True, the code will share future statements with the interactive
2724 2724 shell. It will both be affected by previous __future__ imports, and
2725 2725 any __future__ imports in the code will affect the shell. If False,
2726 2726 __future__ imports are not shared in either direction.
2727 2727 raise_exceptions : bool (False)
2728 2728 If True raise exceptions everywhere. Meant for testing.
2729 2729 """
2730 2730 fname = os.path.abspath(os.path.expanduser(fname))
2731 2731
2732 2732 # Make sure we can open the file
2733 2733 try:
2734 2734 with open(fname):
2735 2735 pass
2736 2736 except:
2737 2737 warn('Could not open file <%s> for safe execution.' % fname)
2738 2738 return
2739 2739
2740 2740 # Find things also in current directory. This is needed to mimic the
2741 2741 # behavior of running a script from the system command line, where
2742 2742 # Python inserts the script's directory into sys.path
2743 2743 dname = os.path.dirname(fname)
2744 2744
2745 2745 def get_cells():
2746 2746 """generator for sequence of code blocks to run"""
2747 2747 if fname.endswith('.ipynb'):
2748 2748 from nbformat import read
2749 2749 nb = read(fname, as_version=4)
2750 2750 if not nb.cells:
2751 2751 return
2752 2752 for cell in nb.cells:
2753 2753 if cell.cell_type == 'code':
2754 2754 yield cell.source
2755 2755 else:
2756 2756 with open(fname) as f:
2757 2757 yield f.read()
2758 2758
2759 2759 with prepended_to_syspath(dname):
2760 2760 try:
2761 2761 for cell in get_cells():
2762 2762 result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
2763 2763 if raise_exceptions:
2764 2764 result.raise_error()
2765 2765 elif not result.success:
2766 2766 break
2767 2767 except:
2768 2768 if raise_exceptions:
2769 2769 raise
2770 2770 self.showtraceback()
2771 2771 warn('Unknown failure executing file: <%s>' % fname)
2772 2772
2773 2773 def safe_run_module(self, mod_name, where):
2774 2774 """A safe version of runpy.run_module().
2775 2775
2776 2776 This version will never throw an exception, but instead print
2777 2777 helpful error messages to the screen.
2778 2778
2779 2779 `SystemExit` exceptions with status code 0 or None are ignored.
2780 2780
2781 2781 Parameters
2782 2782 ----------
2783 2783 mod_name : string
2784 2784 The name of the module to be executed.
2785 2785 where : dict
2786 2786 The globals namespace.
2787 2787 """
2788 2788 try:
2789 2789 try:
2790 2790 where.update(
2791 2791 runpy.run_module(str(mod_name), run_name="__main__",
2792 2792 alter_sys=True)
2793 2793 )
2794 2794 except SystemExit as status:
2795 2795 if status.code:
2796 2796 raise
2797 2797 except:
2798 2798 self.showtraceback()
2799 2799 warn('Unknown failure executing module: <%s>' % mod_name)
2800 2800
2801 2801 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2802 2802 """Run a complete IPython cell.
2803 2803
2804 2804 Parameters
2805 2805 ----------
2806 2806 raw_cell : str
2807 2807 The code (including IPython code such as %magic functions) to run.
2808 2808 store_history : bool
2809 2809 If True, the raw and translated cell will be stored in IPython's
2810 2810 history. For user code calling back into IPython's machinery, this
2811 2811 should be set to False.
2812 2812 silent : bool
2813 2813 If True, avoid side-effects, such as implicit displayhooks and
2814 2814 and logging. silent=True forces store_history=False.
2815 2815 shell_futures : bool
2816 2816 If True, the code will share future statements with the interactive
2817 2817 shell. It will both be affected by previous __future__ imports, and
2818 2818 any __future__ imports in the code will affect the shell. If False,
2819 2819 __future__ imports are not shared in either direction.
2820 2820
2821 2821 Returns
2822 2822 -------
2823 2823 result : :class:`ExecutionResult`
2824 2824 """
2825 2825 result = None
2826 2826 try:
2827 2827 result = self._run_cell(
2828 2828 raw_cell, store_history, silent, shell_futures)
2829 2829 finally:
2830 2830 self.events.trigger('post_execute')
2831 2831 if not silent:
2832 2832 self.events.trigger('post_run_cell', result)
2833 2833 return result
2834 2834
2835 2835 def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
2836 2836 """Internal method to run a complete IPython cell."""
2837 2837 coro = self.run_cell_async(
2838 2838 raw_cell,
2839 2839 store_history=store_history,
2840 2840 silent=silent,
2841 2841 shell_futures=shell_futures,
2842 2842 )
2843 2843
2844 2844 # run_cell_async is async, but may not actually need an eventloop.
2845 2845 # when this is the case, we want to run it using the pseudo_sync_runner
2846 2846 # so that code can invoke eventloops (for example via the %run , and
2847 2847 # `%paste` magic.
2848 2848 if self.should_run_async(raw_cell):
2849 2849 runner = self.loop_runner
2850 2850 else:
2851 2851 runner = _pseudo_sync_runner
2852 2852
2853 2853 try:
2854 2854 return runner(coro)
2855 2855 except BaseException as e:
2856 2856 info = ExecutionInfo(raw_cell, store_history, silent, shell_futures)
2857 2857 result = ExecutionResult(info)
2858 2858 result.error_in_exec = e
2859 2859 self.showtraceback(running_compiled_code=True)
2860 2860 return result
2861 2861 return
2862 2862
2863 2863 def should_run_async(self, raw_cell: str) -> bool:
2864 2864 """Return whether a cell should be run asynchronously via a coroutine runner
2865 2865
2866 2866 Parameters
2867 2867 ----------
2868 2868 raw_cell: str
2869 2869 The code to be executed
2870 2870
2871 2871 Returns
2872 2872 -------
2873 2873 result: bool
2874 2874 Whether the code needs to be run with a coroutine runner or not
2875 2875
2876 2876 .. versionadded: 7.0
2877 2877 """
2878 2878 if not self.autoawait:
2879 2879 return False
2880 2880 try:
2881 2881 cell = self.transform_cell(raw_cell)
2882 2882 except Exception:
2883 2883 # any exception during transform will be raised
2884 2884 # prior to execution
2885 2885 return False
2886 2886 return _should_be_async(cell)
2887 2887
2888 2888 @asyncio.coroutine
2889 2889 def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
2890 2890 """Run a complete IPython cell asynchronously.
2891 2891
2892 2892 Parameters
2893 2893 ----------
2894 2894 raw_cell : str
2895 2895 The code (including IPython code such as %magic functions) to run.
2896 2896 store_history : bool
2897 2897 If True, the raw and translated cell will be stored in IPython's
2898 2898 history. For user code calling back into IPython's machinery, this
2899 2899 should be set to False.
2900 2900 silent : bool
2901 2901 If True, avoid side-effects, such as implicit displayhooks and
2902 2902 and logging. silent=True forces store_history=False.
2903 2903 shell_futures : bool
2904 2904 If True, the code will share future statements with the interactive
2905 2905 shell. It will both be affected by previous __future__ imports, and
2906 2906 any __future__ imports in the code will affect the shell. If False,
2907 2907 __future__ imports are not shared in either direction.
2908 2908
2909 2909 Returns
2910 2910 -------
2911 2911 result : :class:`ExecutionResult`
2912 2912
2913 2913 .. versionadded: 7.0
2914 2914 """
2915 2915 info = ExecutionInfo(
2916 2916 raw_cell, store_history, silent, shell_futures)
2917 2917 result = ExecutionResult(info)
2918 2918
2919 2919 if (not raw_cell) or raw_cell.isspace():
2920 2920 self.last_execution_succeeded = True
2921 2921 self.last_execution_result = result
2922 2922 return result
2923 2923
2924 2924 if silent:
2925 2925 store_history = False
2926 2926
2927 2927 if store_history:
2928 2928 result.execution_count = self.execution_count
2929 2929
2930 2930 def error_before_exec(value):
2931 2931 if store_history:
2932 2932 self.execution_count += 1
2933 2933 result.error_before_exec = value
2934 2934 self.last_execution_succeeded = False
2935 2935 self.last_execution_result = result
2936 2936 return result
2937 2937
2938 2938 self.events.trigger('pre_execute')
2939 2939 if not silent:
2940 2940 self.events.trigger('pre_run_cell', info)
2941 2941
2942 2942 # If any of our input transformation (input_transformer_manager or
2943 2943 # prefilter_manager) raises an exception, we store it in this variable
2944 2944 # so that we can display the error after logging the input and storing
2945 2945 # it in the history.
2946 2946 try:
2947 2947 cell = self.transform_cell(raw_cell)
2948 2948 except Exception:
2949 2949 preprocessing_exc_tuple = sys.exc_info()
2950 2950 cell = raw_cell # cell has to exist so it can be stored/logged
2951 2951 else:
2952 2952 preprocessing_exc_tuple = None
2953 2953
2954 2954 # Store raw and processed history
2955 2955 if store_history:
2956 2956 self.history_manager.store_inputs(self.execution_count,
2957 2957 cell, raw_cell)
2958 2958 if not silent:
2959 2959 self.logger.log(cell, raw_cell)
2960 2960
2961 2961 # Display the exception if input processing failed.
2962 2962 if preprocessing_exc_tuple is not None:
2963 2963 self.showtraceback(preprocessing_exc_tuple)
2964 2964 if store_history:
2965 2965 self.execution_count += 1
2966 2966 return error_before_exec(preprocessing_exc_tuple[2])
2967 2967
2968 2968 # Our own compiler remembers the __future__ environment. If we want to
2969 2969 # run code with a separate __future__ environment, use the default
2970 2970 # compiler
2971 2971 compiler = self.compile if shell_futures else CachingCompiler()
2972 2972
2973 2973 _run_async = False
2974 2974
2975 2975 with self.builtin_trap:
2976 2976 cell_name = self.compile.cache(cell, self.execution_count)
2977 2977
2978 2978 with self.display_trap:
2979 2979 # Compile to bytecode
2980 2980 try:
2981 2981 if self.autoawait and _should_be_async(cell):
2982 2982 # the code AST below will not be user code: we wrap it
2983 2983 # in an `async def`. This will likely make some AST
2984 2984 # transformer below miss some transform opportunity and
2985 2985 # introduce a small coupling to run_code (in which we
2986 2986 # bake some assumptions of what _ast_asyncify returns.
2987 2987 # they are ways around (like grafting part of the ast
2988 2988 # later:
2989 2989 # - Here, return code_ast.body[0].body[1:-1], as well
2990 2990 # as last expression in return statement which is
2991 2991 # the user code part.
2992 2992 # - Let it go through the AST transformers, and graft
2993 2993 # - it back after the AST transform
2994 2994 # But that seem unreasonable, at least while we
2995 2995 # do not need it.
2996 2996 code_ast = _ast_asyncify(cell, 'async-def-wrapper')
2997 2997 _run_async = True
2998 2998 else:
2999 2999 code_ast = compiler.ast_parse(cell, filename=cell_name)
3000 3000 except self.custom_exceptions as e:
3001 3001 etype, value, tb = sys.exc_info()
3002 3002 self.CustomTB(etype, value, tb)
3003 3003 return error_before_exec(e)
3004 3004 except IndentationError as e:
3005 3005 self.showindentationerror()
3006 3006 return error_before_exec(e)
3007 3007 except (OverflowError, SyntaxError, ValueError, TypeError,
3008 3008 MemoryError) as e:
3009 3009 self.showsyntaxerror()
3010 3010 return error_before_exec(e)
3011 3011
3012 3012 # Apply AST transformations
3013 3013 try:
3014 3014 code_ast = self.transform_ast(code_ast)
3015 3015 except InputRejected as e:
3016 3016 self.showtraceback()
3017 3017 return error_before_exec(e)
3018 3018
3019 3019 # Give the displayhook a reference to our ExecutionResult so it
3020 3020 # can fill in the output value.
3021 3021 self.displayhook.exec_result = result
3022 3022
3023 3023 # Execute the user code
3024 3024 interactivity = "none" if silent else self.ast_node_interactivity
3025 3025 if _run_async:
3026 3026 interactivity = 'async'
3027 3027
3028 3028 has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
3029 3029 interactivity=interactivity, compiler=compiler, result=result)
3030 3030
3031 3031 self.last_execution_succeeded = not has_raised
3032 3032 self.last_execution_result = result
3033 3033
3034 3034 # Reset this so later displayed values do not modify the
3035 3035 # ExecutionResult
3036 3036 self.displayhook.exec_result = None
3037 3037
3038 3038 if store_history:
3039 3039 # Write output to the database. Does nothing unless
3040 3040 # history output logging is enabled.
3041 3041 self.history_manager.store_output(self.execution_count)
3042 3042 # Each cell is a *single* input, regardless of how many lines it has
3043 3043 self.execution_count += 1
3044 3044
3045 3045 return result
3046 3046
3047 3047 def transform_cell(self, raw_cell):
3048 3048 """Transform an input cell before parsing it.
3049 3049
3050 3050 Static transformations, implemented in IPython.core.inputtransformer2,
3051 3051 deal with things like ``%magic`` and ``!system`` commands.
3052 3052 These run on all input.
3053 3053 Dynamic transformations, for things like unescaped magics and the exit
3054 3054 autocall, depend on the state of the interpreter.
3055 3055 These only apply to single line inputs.
3056 3056
3057 3057 These string-based transformations are followed by AST transformations;
3058 3058 see :meth:`transform_ast`.
3059 3059 """
3060 3060 # Static input transformations
3061 3061 cell = self.input_transformer_manager.transform_cell(raw_cell)
3062 3062
3063 3063 if len(cell.splitlines()) == 1:
3064 3064 # Dynamic transformations - only applied for single line commands
3065 3065 with self.builtin_trap:
3066 3066 # use prefilter_lines to handle trailing newlines
3067 3067 # restore trailing newline for ast.parse
3068 3068 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
3069 3069
3070 3070 lines = cell.splitlines(keepends=True)
3071 3071 for transform in self.input_transformers_post:
3072 3072 lines = transform(lines)
3073 3073 cell = ''.join(lines)
3074 3074
3075 3075 return cell
3076 3076
3077 3077 def transform_ast(self, node):
3078 3078 """Apply the AST transformations from self.ast_transformers
3079 3079
3080 3080 Parameters
3081 3081 ----------
3082 3082 node : ast.Node
3083 3083 The root node to be transformed. Typically called with the ast.Module
3084 3084 produced by parsing user input.
3085 3085
3086 3086 Returns
3087 3087 -------
3088 3088 An ast.Node corresponding to the node it was called with. Note that it
3089 3089 may also modify the passed object, so don't rely on references to the
3090 3090 original AST.
3091 3091 """
3092 3092 for transformer in self.ast_transformers:
3093 3093 try:
3094 3094 node = transformer.visit(node)
3095 3095 except InputRejected:
3096 3096 # User-supplied AST transformers can reject an input by raising
3097 3097 # an InputRejected. Short-circuit in this case so that we
3098 3098 # don't unregister the transform.
3099 3099 raise
3100 3100 except Exception:
3101 3101 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
3102 3102 self.ast_transformers.remove(transformer)
3103 3103
3104 3104 if self.ast_transformers:
3105 3105 ast.fix_missing_locations(node)
3106 3106 return node
3107 3107
3108 3108 @asyncio.coroutine
3109 3109 def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
3110 3110 compiler=compile, result=None):
3111 3111 """Run a sequence of AST nodes. The execution mode depends on the
3112 3112 interactivity parameter.
3113 3113
3114 3114 Parameters
3115 3115 ----------
3116 3116 nodelist : list
3117 3117 A sequence of AST nodes to run.
3118 3118 cell_name : str
3119 3119 Will be passed to the compiler as the filename of the cell. Typically
3120 3120 the value returned by ip.compile.cache(cell).
3121 3121 interactivity : str
3122 3122 'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
3123 3123 specifying which nodes should be run interactively (displaying output
3124 3124 from expressions). 'last_expr' will run the last node interactively
3125 3125 only if it is an expression (i.e. expressions in loops or other blocks
3126 3126 are not displayed) 'last_expr_or_assign' will run the last expression
3127 3127 or the last assignment. Other values for this parameter will raise a
3128 3128 ValueError.
3129 3129
3130 3130 Experimental value: 'async' Will try to run top level interactive
3131 3131 async/await code in default runner, this will not respect the
3132 3132 interactivty setting and will only run the last node if it is an
3133 3133 expression.
3134 3134
3135 3135 compiler : callable
3136 3136 A function with the same interface as the built-in compile(), to turn
3137 3137 the AST nodes into code objects. Default is the built-in compile().
3138 3138 result : ExecutionResult, optional
3139 3139 An object to store exceptions that occur during execution.
3140 3140
3141 3141 Returns
3142 3142 -------
3143 3143 True if an exception occurred while running code, False if it finished
3144 3144 running.
3145 3145 """
3146 3146 if not nodelist:
3147 3147 return
3148 3148 if interactivity == 'last_expr_or_assign':
3149 3149 if isinstance(nodelist[-1], _assign_nodes):
3150 3150 asg = nodelist[-1]
3151 3151 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
3152 3152 target = asg.targets[0]
3153 3153 elif isinstance(asg, _single_targets_nodes):
3154 3154 target = asg.target
3155 3155 else:
3156 3156 target = None
3157 3157 if isinstance(target, ast.Name):
3158 3158 nnode = ast.Expr(ast.Name(target.id, ast.Load()))
3159 3159 ast.fix_missing_locations(nnode)
3160 3160 nodelist.append(nnode)
3161 3161 interactivity = 'last_expr'
3162 3162
3163 3163 _async = False
3164 3164 if interactivity == 'last_expr':
3165 3165 if isinstance(nodelist[-1], ast.Expr):
3166 3166 interactivity = "last"
3167 3167 else:
3168 3168 interactivity = "none"
3169 3169
3170 3170 if interactivity == 'none':
3171 3171 to_run_exec, to_run_interactive = nodelist, []
3172 3172 elif interactivity == 'last':
3173 3173 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
3174 3174 elif interactivity == 'all':
3175 3175 to_run_exec, to_run_interactive = [], nodelist
3176 3176 elif interactivity == 'async':
3177 3177 _async = True
3178 3178 else:
3179 3179 raise ValueError("Interactivity was %r" % interactivity)
3180 3180 try:
3181 3181 if _async:
3182 3182 # If interactivity is async the semantics of run_code are
3183 3183 # completely different Skip usual machinery.
3184 3184 mod = ast.Module(nodelist)
3185 3185 async_wrapper_code = compiler(mod, 'cell_name', 'exec')
3186 3186 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
3187 3187 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
3188 3188 if (yield from self.run_code(async_code, result, async_=True)):
3189 3189 return True
3190 3190 else:
3191 3191 for i, node in enumerate(to_run_exec):
3192 3192 mod = ast.Module([node])
3193 3193 code = compiler(mod, cell_name, "exec")
3194 3194 if (yield from self.run_code(code, result)):
3195 3195 return True
3196 3196
3197 3197 for i, node in enumerate(to_run_interactive):
3198 3198 mod = ast.Interactive([node])
3199 3199 code = compiler(mod, cell_name, "single")
3200 3200 if (yield from self.run_code(code, result)):
3201 3201 return True
3202 3202
3203 3203 # Flush softspace
3204 3204 if softspace(sys.stdout, 0):
3205 3205 print()
3206 3206
3207 3207 except:
3208 3208 # It's possible to have exceptions raised here, typically by
3209 3209 # compilation of odd code (such as a naked 'return' outside a
3210 3210 # function) that did parse but isn't valid. Typically the exception
3211 3211 # is a SyntaxError, but it's safest just to catch anything and show
3212 3212 # the user a traceback.
3213 3213
3214 3214 # We do only one try/except outside the loop to minimize the impact
3215 3215 # on runtime, and also because if any node in the node list is
3216 3216 # broken, we should stop execution completely.
3217 3217 if result:
3218 3218 result.error_before_exec = sys.exc_info()[1]
3219 3219 self.showtraceback()
3220 3220 return True
3221 3221
3222 3222 return False
3223 3223
3224 3224 def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
3225 3225 """
3226 3226 Evaluate an asynchronous code object using a code runner
3227 3227
3228 3228 Fake asynchronous execution of code_object in a namespace via a proxy namespace.
3229 3229
3230 3230 Returns coroutine object, which can be executed via async loop runner
3231 3231
3232 3232 WARNING: The semantics of `async_exec` are quite different from `exec`,
3233 3233 in particular you can only pass a single namespace. It also return a
3234 3234 handle to the value of the last things returned by code_object.
3235 3235 """
3236 3236
3237 3237 return eval(code_obj, user_ns)
3238 3238
3239 3239 @asyncio.coroutine
3240 3240 def run_code(self, code_obj, result=None, *, async_=False):
3241 3241 """Execute a code object.
3242 3242
3243 3243 When an exception occurs, self.showtraceback() is called to display a
3244 3244 traceback.
3245 3245
3246 3246 Parameters
3247 3247 ----------
3248 3248 code_obj : code object
3249 3249 A compiled code object, to be executed
3250 3250 result : ExecutionResult, optional
3251 3251 An object to store exceptions that occur during execution.
3252 3252 async_ : Bool (Experimental)
3253 3253 Attempt to run top-level asynchronous code in a default loop.
3254 3254
3255 3255 Returns
3256 3256 -------
3257 3257 False : successful execution.
3258 3258 True : an error occurred.
3259 3259 """
3260 3260 # Set our own excepthook in case the user code tries to call it
3261 3261 # directly, so that the IPython crash handler doesn't get triggered
3262 3262 old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
3263 3263
3264 3264 # we save the original sys.excepthook in the instance, in case config
3265 3265 # code (such as magics) needs access to it.
3266 3266 self.sys_excepthook = old_excepthook
3267 3267 outflag = True # happens in more places, so it's easier as default
3268 3268 try:
3269 3269 try:
3270 3270 self.hooks.pre_run_code_hook()
3271 3271 if async_:
3272 3272 last_expr = (yield from self._async_exec(code_obj, self.user_ns))
3273 3273 code = compile('last_expr', 'fake', "single")
3274 3274 exec(code, {'last_expr': last_expr})
3275 3275 else:
3276 3276 exec(code_obj, self.user_global_ns, self.user_ns)
3277 3277 finally:
3278 3278 # Reset our crash handler in place
3279 3279 sys.excepthook = old_excepthook
3280 3280 except SystemExit as e:
3281 3281 if result is not None:
3282 3282 result.error_in_exec = e
3283 3283 self.showtraceback(exception_only=True)
3284 3284 warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
3285 3285 except self.custom_exceptions:
3286 3286 etype, value, tb = sys.exc_info()
3287 3287 if result is not None:
3288 3288 result.error_in_exec = value
3289 3289 self.CustomTB(etype, value, tb)
3290 3290 except:
3291 3291 if result is not None:
3292 3292 result.error_in_exec = sys.exc_info()[1]
3293 3293 self.showtraceback(running_compiled_code=True)
3294 3294 else:
3295 3295 outflag = False
3296 3296 return outflag
3297 3297
3298 3298 # For backwards compatibility
3299 3299 runcode = run_code
3300 3300
3301 3301 def check_complete(self, code: str) -> Tuple[str, str]:
3302 3302 """Return whether a block of code is ready to execute, or should be continued
3303 3303
3304 3304 Parameters
3305 3305 ----------
3306 3306 source : string
3307 3307 Python input code, which can be multiline.
3308 3308
3309 3309 Returns
3310 3310 -------
3311 3311 status : str
3312 3312 One of 'complete', 'incomplete', or 'invalid' if source is not a
3313 3313 prefix of valid code.
3314 3314 indent : str
3315 3315 When status is 'incomplete', this is some whitespace to insert on
3316 3316 the next line of the prompt.
3317 3317 """
3318 3318 status, nspaces = self.input_transformer_manager.check_complete(code)
3319 3319 return status, ' ' * (nspaces or 0)
3320 3320
3321 3321 #-------------------------------------------------------------------------
3322 3322 # Things related to GUI support and pylab
3323 3323 #-------------------------------------------------------------------------
3324 3324
3325 3325 active_eventloop = None
3326 3326
3327 3327 def enable_gui(self, gui=None):
3328 3328 raise NotImplementedError('Implement enable_gui in a subclass')
3329 3329
3330 3330 def enable_matplotlib(self, gui=None):
3331 3331 """Enable interactive matplotlib and inline figure support.
3332 3332
3333 3333 This takes the following steps:
3334 3334
3335 3335 1. select the appropriate eventloop and matplotlib backend
3336 3336 2. set up matplotlib for interactive use with that backend
3337 3337 3. configure formatters for inline figure display
3338 3338 4. enable the selected gui eventloop
3339 3339
3340 3340 Parameters
3341 3341 ----------
3342 3342 gui : optional, string
3343 3343 If given, dictates the choice of matplotlib GUI backend to use
3344 3344 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3345 3345 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3346 3346 matplotlib (as dictated by the matplotlib build-time options plus the
3347 3347 user's matplotlibrc configuration file). Note that not all backends
3348 3348 make sense in all contexts, for example a terminal ipython can't
3349 3349 display figures inline.
3350 3350 """
3351 3351 from IPython.core import pylabtools as pt
3352 3352 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
3353 3353
3354 3354 if gui != 'inline':
3355 3355 # If we have our first gui selection, store it
3356 3356 if self.pylab_gui_select is None:
3357 3357 self.pylab_gui_select = gui
3358 3358 # Otherwise if they are different
3359 3359 elif gui != self.pylab_gui_select:
3360 3360 print('Warning: Cannot change to a different GUI toolkit: %s.'
3361 3361 ' Using %s instead.' % (gui, self.pylab_gui_select))
3362 3362 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
3363 3363
3364 3364 pt.activate_matplotlib(backend)
3365 3365 pt.configure_inline_support(self, backend)
3366 3366
3367 3367 # Now we must activate the gui pylab wants to use, and fix %run to take
3368 3368 # plot updates into account
3369 3369 self.enable_gui(gui)
3370 3370 self.magics_manager.registry['ExecutionMagics'].default_runner = \
3371 3371 pt.mpl_runner(self.safe_execfile)
3372 3372
3373 3373 return gui, backend
3374 3374
3375 3375 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
3376 3376 """Activate pylab support at runtime.
3377 3377
3378 3378 This turns on support for matplotlib, preloads into the interactive
3379 3379 namespace all of numpy and pylab, and configures IPython to correctly
3380 3380 interact with the GUI event loop. The GUI backend to be used can be
3381 3381 optionally selected with the optional ``gui`` argument.
3382 3382
3383 3383 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
3384 3384
3385 3385 Parameters
3386 3386 ----------
3387 3387 gui : optional, string
3388 3388 If given, dictates the choice of matplotlib GUI backend to use
3389 3389 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3390 3390 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3391 3391 matplotlib (as dictated by the matplotlib build-time options plus the
3392 3392 user's matplotlibrc configuration file). Note that not all backends
3393 3393 make sense in all contexts, for example a terminal ipython can't
3394 3394 display figures inline.
3395 3395 import_all : optional, bool, default: True
3396 3396 Whether to do `from numpy import *` and `from pylab import *`
3397 3397 in addition to module imports.
3398 3398 welcome_message : deprecated
3399 3399 This argument is ignored, no welcome message will be displayed.
3400 3400 """
3401 3401 from IPython.core.pylabtools import import_pylab
3402 3402
3403 3403 gui, backend = self.enable_matplotlib(gui)
3404 3404
3405 3405 # We want to prevent the loading of pylab to pollute the user's
3406 3406 # namespace as shown by the %who* magics, so we execute the activation
3407 3407 # code in an empty namespace, and we update *both* user_ns and
3408 3408 # user_ns_hidden with this information.
3409 3409 ns = {}
3410 3410 import_pylab(ns, import_all)
3411 3411 # warn about clobbered names
3412 3412 ignored = {"__builtins__"}
3413 3413 both = set(ns).intersection(self.user_ns).difference(ignored)
3414 3414 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
3415 3415 self.user_ns.update(ns)
3416 3416 self.user_ns_hidden.update(ns)
3417 3417 return gui, backend, clobbered
3418 3418
3419 3419 #-------------------------------------------------------------------------
3420 3420 # Utilities
3421 3421 #-------------------------------------------------------------------------
3422 3422
3423 3423 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
3424 3424 """Expand python variables in a string.
3425 3425
3426 3426 The depth argument indicates how many frames above the caller should
3427 3427 be walked to look for the local namespace where to expand variables.
3428 3428
3429 3429 The global namespace for expansion is always the user's interactive
3430 3430 namespace.
3431 3431 """
3432 3432 ns = self.user_ns.copy()
3433 3433 try:
3434 3434 frame = sys._getframe(depth+1)
3435 3435 except ValueError:
3436 3436 # This is thrown if there aren't that many frames on the stack,
3437 3437 # e.g. if a script called run_line_magic() directly.
3438 3438 pass
3439 3439 else:
3440 3440 ns.update(frame.f_locals)
3441 3441
3442 3442 try:
3443 3443 # We have to use .vformat() here, because 'self' is a valid and common
3444 3444 # name, and expanding **ns for .format() would make it collide with
3445 3445 # the 'self' argument of the method.
3446 3446 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
3447 3447 except Exception:
3448 3448 # if formatter couldn't format, just let it go untransformed
3449 3449 pass
3450 3450 return cmd
3451 3451
3452 3452 def mktempfile(self, data=None, prefix='ipython_edit_'):
3453 3453 """Make a new tempfile and return its filename.
3454 3454
3455 3455 This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
3456 3456 but it registers the created filename internally so ipython cleans it up
3457 3457 at exit time.
3458 3458
3459 3459 Optional inputs:
3460 3460
3461 3461 - data(None): if data is given, it gets written out to the temp file
3462 3462 immediately, and the file is closed again."""
3463 3463
3464 3464 dirname = tempfile.mkdtemp(prefix=prefix)
3465 3465 self.tempdirs.append(dirname)
3466 3466
3467 3467 handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
3468 3468 os.close(handle) # On Windows, there can only be one open handle on a file
3469 3469 self.tempfiles.append(filename)
3470 3470
3471 3471 if data:
3472 tmp_file = open(filename,'w')
3473 tmp_file.write(data)
3474 tmp_file.close()
3472 with open(filename, 'w') as tmp_file:
3473 tmp_file.write(data)
3475 3474 return filename
3476 3475
3477 3476 @undoc
3478 3477 def write(self,data):
3479 3478 """DEPRECATED: Write a string to the default output"""
3480 3479 warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
3481 3480 DeprecationWarning, stacklevel=2)
3482 3481 sys.stdout.write(data)
3483 3482
3484 3483 @undoc
3485 3484 def write_err(self,data):
3486 3485 """DEPRECATED: Write a string to the default error output"""
3487 3486 warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
3488 3487 DeprecationWarning, stacklevel=2)
3489 3488 sys.stderr.write(data)
3490 3489
3491 3490 def ask_yes_no(self, prompt, default=None, interrupt=None):
3492 3491 if self.quiet:
3493 3492 return True
3494 3493 return ask_yes_no(prompt,default,interrupt)
3495 3494
3496 3495 def show_usage(self):
3497 3496 """Show a usage message"""
3498 3497 page.page(IPython.core.usage.interactive_usage)
3499 3498
3500 3499 def extract_input_lines(self, range_str, raw=False):
3501 3500 """Return as a string a set of input history slices.
3502 3501
3503 3502 Parameters
3504 3503 ----------
3505 3504 range_str : string
3506 3505 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3507 3506 since this function is for use by magic functions which get their
3508 3507 arguments as strings. The number before the / is the session
3509 3508 number: ~n goes n back from the current session.
3510 3509
3511 3510 raw : bool, optional
3512 3511 By default, the processed input is used. If this is true, the raw
3513 3512 input history is used instead.
3514 3513
3515 3514 Notes
3516 3515 -----
3517 3516
3518 3517 Slices can be described with two notations:
3519 3518
3520 3519 * ``N:M`` -> standard python form, means including items N...(M-1).
3521 3520 * ``N-M`` -> include items N..M (closed endpoint).
3522 3521 """
3523 3522 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3524 3523 return "\n".join(x for _, _, x in lines)
3525 3524
3526 3525 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
3527 3526 """Get a code string from history, file, url, or a string or macro.
3528 3527
3529 3528 This is mainly used by magic functions.
3530 3529
3531 3530 Parameters
3532 3531 ----------
3533 3532
3534 3533 target : str
3535 3534
3536 3535 A string specifying code to retrieve. This will be tried respectively
3537 3536 as: ranges of input history (see %history for syntax), url,
3538 3537 corresponding .py file, filename, or an expression evaluating to a
3539 3538 string or Macro in the user namespace.
3540 3539
3541 3540 raw : bool
3542 3541 If true (default), retrieve raw history. Has no effect on the other
3543 3542 retrieval mechanisms.
3544 3543
3545 3544 py_only : bool (default False)
3546 3545 Only try to fetch python code, do not try alternative methods to decode file
3547 3546 if unicode fails.
3548 3547
3549 3548 Returns
3550 3549 -------
3551 3550 A string of code.
3552 3551
3553 3552 ValueError is raised if nothing is found, and TypeError if it evaluates
3554 3553 to an object of another type. In each case, .args[0] is a printable
3555 3554 message.
3556 3555 """
3557 3556 code = self.extract_input_lines(target, raw=raw) # Grab history
3558 3557 if code:
3559 3558 return code
3560 3559 try:
3561 3560 if target.startswith(('http://', 'https://')):
3562 3561 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
3563 3562 except UnicodeDecodeError:
3564 3563 if not py_only :
3565 3564 # Deferred import
3566 3565 from urllib.request import urlopen
3567 3566 response = urlopen(target)
3568 3567 return response.read().decode('latin1')
3569 3568 raise ValueError(("'%s' seem to be unreadable.") % target)
3570 3569
3571 3570 potential_target = [target]
3572 3571 try :
3573 3572 potential_target.insert(0,get_py_filename(target))
3574 3573 except IOError:
3575 3574 pass
3576 3575
3577 3576 for tgt in potential_target :
3578 3577 if os.path.isfile(tgt): # Read file
3579 3578 try :
3580 3579 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3581 3580 except UnicodeDecodeError :
3582 3581 if not py_only :
3583 3582 with io_open(tgt,'r', encoding='latin1') as f :
3584 3583 return f.read()
3585 3584 raise ValueError(("'%s' seem to be unreadable.") % target)
3586 3585 elif os.path.isdir(os.path.expanduser(tgt)):
3587 3586 raise ValueError("'%s' is a directory, not a regular file." % target)
3588 3587
3589 3588 if search_ns:
3590 3589 # Inspect namespace to load object source
3591 3590 object_info = self.object_inspect(target, detail_level=1)
3592 3591 if object_info['found'] and object_info['source']:
3593 3592 return object_info['source']
3594 3593
3595 3594 try: # User namespace
3596 3595 codeobj = eval(target, self.user_ns)
3597 3596 except Exception:
3598 3597 raise ValueError(("'%s' was not found in history, as a file, url, "
3599 3598 "nor in the user namespace.") % target)
3600 3599
3601 3600 if isinstance(codeobj, str):
3602 3601 return codeobj
3603 3602 elif isinstance(codeobj, Macro):
3604 3603 return codeobj.value
3605 3604
3606 3605 raise TypeError("%s is neither a string nor a macro." % target,
3607 3606 codeobj)
3608 3607
3609 3608 #-------------------------------------------------------------------------
3610 3609 # Things related to IPython exiting
3611 3610 #-------------------------------------------------------------------------
3612 3611 def atexit_operations(self):
3613 3612 """This will be executed at the time of exit.
3614 3613
3615 3614 Cleanup operations and saving of persistent data that is done
3616 3615 unconditionally by IPython should be performed here.
3617 3616
3618 3617 For things that may depend on startup flags or platform specifics (such
3619 3618 as having readline or not), register a separate atexit function in the
3620 3619 code that has the appropriate information, rather than trying to
3621 3620 clutter
3622 3621 """
3623 3622 # Close the history session (this stores the end time and line count)
3624 3623 # this must be *before* the tempfile cleanup, in case of temporary
3625 3624 # history db
3626 3625 self.history_manager.end_session()
3627 3626
3628 3627 # Cleanup all tempfiles and folders left around
3629 3628 for tfile in self.tempfiles:
3630 3629 try:
3631 3630 os.unlink(tfile)
3632 3631 except OSError:
3633 3632 pass
3634 3633
3635 3634 for tdir in self.tempdirs:
3636 3635 try:
3637 3636 os.rmdir(tdir)
3638 3637 except OSError:
3639 3638 pass
3640 3639
3641 3640 # Clear all user namespaces to release all references cleanly.
3642 3641 self.reset(new_session=False)
3643 3642
3644 3643 # Run user hooks
3645 3644 self.hooks.shutdown_hook()
3646 3645
3647 3646 def cleanup(self):
3648 3647 self.restore_sys_module_state()
3649 3648
3650 3649
3651 3650 # Overridden in terminal subclass to change prompts
3652 3651 def switch_doctest_mode(self, mode):
3653 3652 pass
3654 3653
3655 3654
3656 3655 class InteractiveShellABC(metaclass=abc.ABCMeta):
3657 3656 """An abstract base class for InteractiveShell."""
3658 3657
3659 3658 InteractiveShellABC.register(InteractiveShell)
@@ -1,731 +1,732 b''
1 1 """Implementation of code management magic functions.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (c) 2012 The IPython Development Team.
5 5 #
6 6 # Distributed under the terms of the Modified BSD License.
7 7 #
8 8 # The full license is in the file COPYING.txt, distributed with this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Imports
13 13 #-----------------------------------------------------------------------------
14 14
15 15 # Stdlib
16 16 import inspect
17 17 import io
18 18 import os
19 19 import re
20 20 import sys
21 21 import ast
22 22 from itertools import chain
23 23 from urllib.request import urlopen
24 24 from urllib.parse import urlencode
25 25
26 26 # Our own packages
27 27 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
28 28 from IPython.core.macro import Macro
29 29 from IPython.core.magic import Magics, magics_class, line_magic
30 30 from IPython.core.oinspect import find_file, find_source_lines
31 31 from IPython.testing.skipdoctest import skip_doctest
32 32 from IPython.utils import py3compat
33 33 from IPython.utils.contexts import preserve_keys
34 34 from IPython.utils.path import get_py_filename
35 35 from warnings import warn
36 36 from logging import error
37 37 from IPython.utils.text import get_text_list
38 38
39 39 #-----------------------------------------------------------------------------
40 40 # Magic implementation classes
41 41 #-----------------------------------------------------------------------------
42 42
43 43 # Used for exception handling in magic_edit
44 44 class MacroToEdit(ValueError): pass
45 45
46 46 ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
47 47
48 48 # To match, e.g. 8-10 1:5 :10 3-
49 49 range_re = re.compile(r"""
50 50 (?P<start>\d+)?
51 51 ((?P<sep>[\-:])
52 52 (?P<end>\d+)?)?
53 53 $""", re.VERBOSE)
54 54
55 55
56 56 def extract_code_ranges(ranges_str):
57 57 """Turn a string of range for %%load into 2-tuples of (start, stop)
58 58 ready to use as a slice of the content split by lines.
59 59
60 60 Examples
61 61 --------
62 62 list(extract_input_ranges("5-10 2"))
63 63 [(4, 10), (1, 2)]
64 64 """
65 65 for range_str in ranges_str.split():
66 66 rmatch = range_re.match(range_str)
67 67 if not rmatch:
68 68 continue
69 69 sep = rmatch.group("sep")
70 70 start = rmatch.group("start")
71 71 end = rmatch.group("end")
72 72
73 73 if sep == '-':
74 74 start = int(start) - 1 if start else None
75 75 end = int(end) if end else None
76 76 elif sep == ':':
77 77 start = int(start) - 1 if start else None
78 78 end = int(end) - 1 if end else None
79 79 else:
80 80 end = int(start)
81 81 start = int(start) - 1
82 82 yield (start, end)
83 83
84 84
85 85 def extract_symbols(code, symbols):
86 86 """
87 87 Return a tuple (blocks, not_found)
88 88 where ``blocks`` is a list of code fragments
89 89 for each symbol parsed from code, and ``not_found`` are
90 90 symbols not found in the code.
91 91
92 92 For example::
93 93
94 94 In [1]: code = '''a = 10
95 95 ...: def b(): return 42
96 96 ...: class A: pass'''
97 97
98 98 In [2]: extract_symbols(code, 'A,b,z')
99 99 Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z'])
100 100 """
101 101 symbols = symbols.split(',')
102 102
103 103 # this will raise SyntaxError if code isn't valid Python
104 104 py_code = ast.parse(code)
105 105
106 106 marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body]
107 107 code = code.split('\n')
108 108
109 109 symbols_lines = {}
110 110
111 111 # we already know the start_lineno of each symbol (marks).
112 112 # To find each end_lineno, we traverse in reverse order until each
113 113 # non-blank line
114 114 end = len(code)
115 115 for name, start in reversed(marks):
116 116 while not code[end - 1].strip():
117 117 end -= 1
118 118 if name:
119 119 symbols_lines[name] = (start - 1, end)
120 120 end = start - 1
121 121
122 122 # Now symbols_lines is a map
123 123 # {'symbol_name': (start_lineno, end_lineno), ...}
124 124
125 125 # fill a list with chunks of codes for each requested symbol
126 126 blocks = []
127 127 not_found = []
128 128 for symbol in symbols:
129 129 if symbol in symbols_lines:
130 130 start, end = symbols_lines[symbol]
131 131 blocks.append('\n'.join(code[start:end]) + '\n')
132 132 else:
133 133 not_found.append(symbol)
134 134
135 135 return blocks, not_found
136 136
137 137 def strip_initial_indent(lines):
138 138 """For %load, strip indent from lines until finding an unindented line.
139 139
140 140 https://github.com/ipython/ipython/issues/9775
141 141 """
142 142 indent_re = re.compile(r'\s+')
143 143
144 144 it = iter(lines)
145 145 first_line = next(it)
146 146 indent_match = indent_re.match(first_line)
147 147
148 148 if indent_match:
149 149 # First line was indented
150 150 indent = indent_match.group()
151 151 yield first_line[len(indent):]
152 152
153 153 for line in it:
154 154 if line.startswith(indent):
155 155 yield line[len(indent):]
156 156 else:
157 157 # Less indented than the first line - stop dedenting
158 158 yield line
159 159 break
160 160 else:
161 161 yield first_line
162 162
163 163 # Pass the remaining lines through without dedenting
164 164 for line in it:
165 165 yield line
166 166
167 167
168 168 class InteractivelyDefined(Exception):
169 169 """Exception for interactively defined variable in magic_edit"""
170 170 def __init__(self, index):
171 171 self.index = index
172 172
173 173
174 174 @magics_class
175 175 class CodeMagics(Magics):
176 176 """Magics related to code management (loading, saving, editing, ...)."""
177 177
178 178 def __init__(self, *args, **kwargs):
179 179 self._knowntemps = set()
180 180 super(CodeMagics, self).__init__(*args, **kwargs)
181 181
182 182 @line_magic
183 183 def save(self, parameter_s=''):
184 184 """Save a set of lines or a macro to a given filename.
185 185
186 186 Usage:\\
187 187 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
188 188
189 189 Options:
190 190
191 191 -r: use 'raw' input. By default, the 'processed' history is used,
192 192 so that magics are loaded in their transformed version to valid
193 193 Python. If this option is given, the raw input as typed as the
194 194 command line is used instead.
195 195
196 196 -f: force overwrite. If file exists, %save will prompt for overwrite
197 197 unless -f is given.
198 198
199 199 -a: append to the file instead of overwriting it.
200 200
201 201 This function uses the same syntax as %history for input ranges,
202 202 then saves the lines to the filename you specify.
203 203
204 204 It adds a '.py' extension to the file if you don't do so yourself, and
205 205 it asks for confirmation before overwriting existing files.
206 206
207 207 If `-r` option is used, the default extension is `.ipy`.
208 208 """
209 209
210 210 opts,args = self.parse_options(parameter_s,'fra',mode='list')
211 211 if not args:
212 212 raise UsageError('Missing filename.')
213 213 raw = 'r' in opts
214 214 force = 'f' in opts
215 215 append = 'a' in opts
216 216 mode = 'a' if append else 'w'
217 217 ext = u'.ipy' if raw else u'.py'
218 218 fname, codefrom = args[0], " ".join(args[1:])
219 219 if not fname.endswith((u'.py',u'.ipy')):
220 220 fname += ext
221 221 file_exists = os.path.isfile(fname)
222 222 if file_exists and not force and not append:
223 223 try:
224 224 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
225 225 except StdinNotImplementedError:
226 226 print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s))
227 227 return
228 228 if not overwrite :
229 229 print('Operation cancelled.')
230 230 return
231 231 try:
232 232 cmds = self.shell.find_user_code(codefrom,raw)
233 233 except (TypeError, ValueError) as e:
234 234 print(e.args[0])
235 235 return
236 236 out = py3compat.cast_unicode(cmds)
237 237 with io.open(fname, mode, encoding="utf-8") as f:
238 238 if not file_exists or not append:
239 239 f.write(u"# coding: utf-8\n")
240 240 f.write(out)
241 241 # make sure we end on a newline
242 242 if not out.endswith(u'\n'):
243 243 f.write(u'\n')
244 244 print('The following commands were written to file `%s`:' % fname)
245 245 print(cmds)
246 246
247 247 @line_magic
248 248 def pastebin(self, parameter_s=''):
249 249 """Upload code to dpaste's paste bin, returning the URL.
250 250
251 251 Usage:\\
252 252 %pastebin [-d "Custom description"] 1-7
253 253
254 254 The argument can be an input history range, a filename, or the name of a
255 255 string or macro.
256 256
257 257 Options:
258 258
259 259 -d: Pass a custom description for the gist. The default will say
260 260 "Pasted from IPython".
261 261 """
262 262 opts, args = self.parse_options(parameter_s, 'd:')
263 263
264 264 try:
265 265 code = self.shell.find_user_code(args)
266 266 except (ValueError, TypeError) as e:
267 267 print(e.args[0])
268 268 return
269 269
270 270 post_data = urlencode({
271 271 "title": opts.get('d', "Pasted from IPython"),
272 272 "syntax": "python3",
273 273 "content": code
274 274 }).encode('utf-8')
275 275
276 276 response = urlopen("http://dpaste.com/api/v2/", post_data)
277 277 return response.headers.get('Location')
278 278
279 279 @line_magic
280 280 def loadpy(self, arg_s):
281 281 """Alias of `%load`
282 282
283 283 `%loadpy` has gained some flexibility and dropped the requirement of a `.py`
284 284 extension. So it has been renamed simply into %load. You can look at
285 285 `%load`'s docstring for more info.
286 286 """
287 287 self.load(arg_s)
288 288
289 289 @line_magic
290 290 def load(self, arg_s):
291 291 """Load code into the current frontend.
292 292
293 293 Usage:\\
294 294 %load [options] source
295 295
296 296 where source can be a filename, URL, input history range, macro, or
297 297 element in the user namespace
298 298
299 299 Options:
300 300
301 301 -r <lines>: Specify lines or ranges of lines to load from the source.
302 302 Ranges could be specified as x-y (x..y) or in python-style x:y
303 303 (x..(y-1)). Both limits x and y can be left blank (meaning the
304 304 beginning and end of the file, respectively).
305 305
306 306 -s <symbols>: Specify function or classes to load from python source.
307 307
308 308 -y : Don't ask confirmation for loading source above 200 000 characters.
309 309
310 310 -n : Include the user's namespace when searching for source code.
311 311
312 312 This magic command can either take a local filename, a URL, an history
313 313 range (see %history) or a macro as argument, it will prompt for
314 314 confirmation before loading source with more than 200 000 characters, unless
315 315 -y flag is passed or if the frontend does not support raw_input::
316 316
317 317 %load myscript.py
318 318 %load 7-27
319 319 %load myMacro
320 320 %load http://www.example.com/myscript.py
321 321 %load -r 5-10 myscript.py
322 322 %load -r 10-20,30,40: foo.py
323 323 %load -s MyClass,wonder_function myscript.py
324 324 %load -n MyClass
325 325 %load -n my_module.wonder_function
326 326 """
327 327 opts,args = self.parse_options(arg_s,'yns:r:')
328 328
329 329 if not args:
330 330 raise UsageError('Missing filename, URL, input history range, '
331 331 'macro, or element in the user namespace.')
332 332
333 333 search_ns = 'n' in opts
334 334
335 335 contents = self.shell.find_user_code(args, search_ns=search_ns)
336 336
337 337 if 's' in opts:
338 338 try:
339 339 blocks, not_found = extract_symbols(contents, opts['s'])
340 340 except SyntaxError:
341 341 # non python code
342 342 error("Unable to parse the input as valid Python code")
343 343 return
344 344
345 345 if len(not_found) == 1:
346 346 warn('The symbol `%s` was not found' % not_found[0])
347 347 elif len(not_found) > 1:
348 348 warn('The symbols %s were not found' % get_text_list(not_found,
349 349 wrap_item_with='`')
350 350 )
351 351
352 352 contents = '\n'.join(blocks)
353 353
354 354 if 'r' in opts:
355 355 ranges = opts['r'].replace(',', ' ')
356 356 lines = contents.split('\n')
357 357 slices = extract_code_ranges(ranges)
358 358 contents = [lines[slice(*slc)] for slc in slices]
359 359 contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents)))
360 360
361 361 l = len(contents)
362 362
363 363 # 200 000 is ~ 2500 full 80 character lines
364 364 # so in average, more than 5000 lines
365 365 if l > 200000 and 'y' not in opts:
366 366 try:
367 367 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
368 368 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
369 369 except StdinNotImplementedError:
370 370 #assume yes if raw input not implemented
371 371 ans = True
372 372
373 373 if ans is False :
374 374 print('Operation cancelled.')
375 375 return
376 376
377 377 contents = "# %load {}\n".format(arg_s) + contents
378 378
379 379 self.shell.set_next_input(contents, replace=True)
380 380
381 381 @staticmethod
382 382 def _find_edit_target(shell, args, opts, last_call):
383 383 """Utility method used by magic_edit to find what to edit."""
384 384
385 385 def make_filename(arg):
386 386 "Make a filename from the given args"
387 387 try:
388 388 filename = get_py_filename(arg)
389 389 except IOError:
390 390 # If it ends with .py but doesn't already exist, assume we want
391 391 # a new file.
392 392 if arg.endswith('.py'):
393 393 filename = arg
394 394 else:
395 395 filename = None
396 396 return filename
397 397
398 398 # Set a few locals from the options for convenience:
399 399 opts_prev = 'p' in opts
400 400 opts_raw = 'r' in opts
401 401
402 402 # custom exceptions
403 403 class DataIsObject(Exception): pass
404 404
405 405 # Default line number value
406 406 lineno = opts.get('n',None)
407 407
408 408 if opts_prev:
409 409 args = '_%s' % last_call[0]
410 410 if args not in shell.user_ns:
411 411 args = last_call[1]
412 412
413 413 # by default this is done with temp files, except when the given
414 414 # arg is a filename
415 415 use_temp = True
416 416
417 417 data = ''
418 418
419 419 # First, see if the arguments should be a filename.
420 420 filename = make_filename(args)
421 421 if filename:
422 422 use_temp = False
423 423 elif args:
424 424 # Mode where user specifies ranges of lines, like in %macro.
425 425 data = shell.extract_input_lines(args, opts_raw)
426 426 if not data:
427 427 try:
428 428 # Load the parameter given as a variable. If not a string,
429 429 # process it as an object instead (below)
430 430
431 431 #print '*** args',args,'type',type(args) # dbg
432 432 data = eval(args, shell.user_ns)
433 433 if not isinstance(data, str):
434 434 raise DataIsObject
435 435
436 436 except (NameError,SyntaxError):
437 437 # given argument is not a variable, try as a filename
438 438 filename = make_filename(args)
439 439 if filename is None:
440 440 warn("Argument given (%s) can't be found as a variable "
441 441 "or as a filename." % args)
442 442 return (None, None, None)
443 443 use_temp = False
444 444
445 445 except DataIsObject:
446 446 # macros have a special edit function
447 447 if isinstance(data, Macro):
448 448 raise MacroToEdit(data)
449 449
450 450 # For objects, try to edit the file where they are defined
451 451 filename = find_file(data)
452 452 if filename:
453 453 if 'fakemodule' in filename.lower() and \
454 454 inspect.isclass(data):
455 455 # class created by %edit? Try to find source
456 456 # by looking for method definitions instead, the
457 457 # __module__ in those classes is FakeModule.
458 458 attrs = [getattr(data, aname) for aname in dir(data)]
459 459 for attr in attrs:
460 460 if not inspect.ismethod(attr):
461 461 continue
462 462 filename = find_file(attr)
463 463 if filename and \
464 464 'fakemodule' not in filename.lower():
465 465 # change the attribute to be the edit
466 466 # target instead
467 467 data = attr
468 468 break
469 469
470 470 m = ipython_input_pat.match(os.path.basename(filename))
471 471 if m:
472 472 raise InteractivelyDefined(int(m.groups()[0]))
473 473
474 474 datafile = 1
475 475 if filename is None:
476 476 filename = make_filename(args)
477 477 datafile = 1
478 478 if filename is not None:
479 479 # only warn about this if we get a real name
480 480 warn('Could not find file where `%s` is defined.\n'
481 481 'Opening a file named `%s`' % (args, filename))
482 482 # Now, make sure we can actually read the source (if it was
483 483 # in a temp file it's gone by now).
484 484 if datafile:
485 485 if lineno is None:
486 486 lineno = find_source_lines(data)
487 487 if lineno is None:
488 488 filename = make_filename(args)
489 489 if filename is None:
490 490 warn('The file where `%s` was defined '
491 491 'cannot be read or found.' % data)
492 492 return (None, None, None)
493 493 use_temp = False
494 494
495 495 if use_temp:
496 496 filename = shell.mktempfile(data)
497 497 print('IPython will make a temporary file named:',filename)
498 498
499 499 # use last_call to remember the state of the previous call, but don't
500 500 # let it be clobbered by successive '-p' calls.
501 501 try:
502 502 last_call[0] = shell.displayhook.prompt_count
503 503 if not opts_prev:
504 504 last_call[1] = args
505 505 except:
506 506 pass
507 507
508 508
509 509 return filename, lineno, use_temp
510 510
511 511 def _edit_macro(self,mname,macro):
512 512 """open an editor with the macro data in a file"""
513 513 filename = self.shell.mktempfile(macro.value)
514 514 self.shell.hooks.editor(filename)
515 515
516 516 # and make a new macro object, to replace the old one
517 517 with open(filename) as mfile:
518 518 mvalue = mfile.read()
519 519 self.shell.user_ns[mname] = Macro(mvalue)
520 520
521 521 @skip_doctest
522 522 @line_magic
523 523 def edit(self, parameter_s='',last_call=['','']):
524 524 """Bring up an editor and execute the resulting code.
525 525
526 526 Usage:
527 527 %edit [options] [args]
528 528
529 529 %edit runs IPython's editor hook. The default version of this hook is
530 530 set to call the editor specified by your $EDITOR environment variable.
531 531 If this isn't found, it will default to vi under Linux/Unix and to
532 532 notepad under Windows. See the end of this docstring for how to change
533 533 the editor hook.
534 534
535 535 You can also set the value of this editor via the
536 536 ``TerminalInteractiveShell.editor`` option in your configuration file.
537 537 This is useful if you wish to use a different editor from your typical
538 538 default with IPython (and for Windows users who typically don't set
539 539 environment variables).
540 540
541 541 This command allows you to conveniently edit multi-line code right in
542 542 your IPython session.
543 543
544 544 If called without arguments, %edit opens up an empty editor with a
545 545 temporary file and will execute the contents of this file when you
546 546 close it (don't forget to save it!).
547 547
548 548
549 549 Options:
550 550
551 551 -n <number>: open the editor at a specified line number. By default,
552 552 the IPython editor hook uses the unix syntax 'editor +N filename', but
553 553 you can configure this by providing your own modified hook if your
554 554 favorite editor supports line-number specifications with a different
555 555 syntax.
556 556
557 557 -p: this will call the editor with the same data as the previous time
558 558 it was used, regardless of how long ago (in your current session) it
559 559 was.
560 560
561 561 -r: use 'raw' input. This option only applies to input taken from the
562 562 user's history. By default, the 'processed' history is used, so that
563 563 magics are loaded in their transformed version to valid Python. If
564 564 this option is given, the raw input as typed as the command line is
565 565 used instead. When you exit the editor, it will be executed by
566 566 IPython's own processor.
567 567
568 568 -x: do not execute the edited code immediately upon exit. This is
569 569 mainly useful if you are editing programs which need to be called with
570 570 command line arguments, which you can then do using %run.
571 571
572 572
573 573 Arguments:
574 574
575 575 If arguments are given, the following possibilities exist:
576 576
577 577 - If the argument is a filename, IPython will load that into the
578 578 editor. It will execute its contents with execfile() when you exit,
579 579 loading any code in the file into your interactive namespace.
580 580
581 581 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
582 582 The syntax is the same as in the %history magic.
583 583
584 584 - If the argument is a string variable, its contents are loaded
585 585 into the editor. You can thus edit any string which contains
586 586 python code (including the result of previous edits).
587 587
588 588 - If the argument is the name of an object (other than a string),
589 589 IPython will try to locate the file where it was defined and open the
590 590 editor at the point where it is defined. You can use `%edit function`
591 591 to load an editor exactly at the point where 'function' is defined,
592 592 edit it and have the file be executed automatically.
593 593
594 594 - If the object is a macro (see %macro for details), this opens up your
595 595 specified editor with a temporary file containing the macro's data.
596 596 Upon exit, the macro is reloaded with the contents of the file.
597 597
598 598 Note: opening at an exact line is only supported under Unix, and some
599 599 editors (like kedit and gedit up to Gnome 2.8) do not understand the
600 600 '+NUMBER' parameter necessary for this feature. Good editors like
601 601 (X)Emacs, vi, jed, pico and joe all do.
602 602
603 603 After executing your code, %edit will return as output the code you
604 604 typed in the editor (except when it was an existing file). This way
605 605 you can reload the code in further invocations of %edit as a variable,
606 606 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
607 607 the output.
608 608
609 609 Note that %edit is also available through the alias %ed.
610 610
611 611 This is an example of creating a simple function inside the editor and
612 612 then modifying it. First, start up the editor::
613 613
614 614 In [1]: edit
615 615 Editing... done. Executing edited code...
616 616 Out[1]: 'def foo():\\n print "foo() was defined in an editing
617 617 session"\\n'
618 618
619 619 We can then call the function foo()::
620 620
621 621 In [2]: foo()
622 622 foo() was defined in an editing session
623 623
624 624 Now we edit foo. IPython automatically loads the editor with the
625 625 (temporary) file where foo() was previously defined::
626 626
627 627 In [3]: edit foo
628 628 Editing... done. Executing edited code...
629 629
630 630 And if we call foo() again we get the modified version::
631 631
632 632 In [4]: foo()
633 633 foo() has now been changed!
634 634
635 635 Here is an example of how to edit a code snippet successive
636 636 times. First we call the editor::
637 637
638 638 In [5]: edit
639 639 Editing... done. Executing edited code...
640 640 hello
641 641 Out[5]: "print 'hello'\\n"
642 642
643 643 Now we call it again with the previous output (stored in _)::
644 644
645 645 In [6]: edit _
646 646 Editing... done. Executing edited code...
647 647 hello world
648 648 Out[6]: "print 'hello world'\\n"
649 649
650 650 Now we call it with the output #8 (stored in _8, also as Out[8])::
651 651
652 652 In [7]: edit _8
653 653 Editing... done. Executing edited code...
654 654 hello again
655 655 Out[7]: "print 'hello again'\\n"
656 656
657 657
658 658 Changing the default editor hook:
659 659
660 660 If you wish to write your own editor hook, you can put it in a
661 661 configuration file which you load at startup time. The default hook
662 662 is defined in the IPython.core.hooks module, and you can use that as a
663 663 starting example for further modifications. That file also has
664 664 general instructions on how to set a new hook for use once you've
665 665 defined it."""
666 666 opts,args = self.parse_options(parameter_s,'prxn:')
667 667
668 668 try:
669 669 filename, lineno, is_temp = self._find_edit_target(self.shell,
670 670 args, opts, last_call)
671 671 except MacroToEdit as e:
672 672 self._edit_macro(args, e.args[0])
673 673 return
674 674 except InteractivelyDefined as e:
675 675 print("Editing In[%i]" % e.index)
676 676 args = str(e.index)
677 677 filename, lineno, is_temp = self._find_edit_target(self.shell,
678 678 args, opts, last_call)
679 679 if filename is None:
680 680 # nothing was found, warnings have already been issued,
681 681 # just give up.
682 682 return
683 683
684 684 if is_temp:
685 685 self._knowntemps.add(filename)
686 686 elif (filename in self._knowntemps):
687 687 is_temp = True
688 688
689 689
690 690 # do actual editing here
691 691 print('Editing...', end=' ')
692 692 sys.stdout.flush()
693 693 try:
694 694 # Quote filenames that may have spaces in them
695 695 if ' ' in filename:
696 696 filename = "'%s'" % filename
697 697 self.shell.hooks.editor(filename,lineno)
698 698 except TryNext:
699 699 warn('Could not open editor')
700 700 return
701 701
702 702 # XXX TODO: should this be generalized for all string vars?
703 703 # For now, this is special-cased to blocks created by cpaste
704 704 if args.strip() == 'pasted_block':
705 705 with open(filename, 'r') as f:
706 706 self.shell.user_ns['pasted_block'] = f.read()
707 707
708 708 if 'x' in opts: # -x prevents actual execution
709 709 print()
710 710 else:
711 711 print('done. Executing edited code...')
712 712 with preserve_keys(self.shell.user_ns, '__file__'):
713 713 if not is_temp:
714 714 self.shell.user_ns['__file__'] = filename
715 715 if 'r' in opts: # Untranslated IPython code
716 716 with open(filename, 'r') as f:
717 717 source = f.read()
718 718 self.shell.run_cell(source, store_history=False)
719 719 else:
720 720 self.shell.safe_execfile(filename, self.shell.user_ns,
721 721 self.shell.user_ns)
722 722
723 723 if is_temp:
724 724 try:
725 return open(filename).read()
725 with open(filename) as f:
726 return f.read()
726 727 except IOError as msg:
727 728 if msg.filename == filename:
728 729 warn('File not found. Did you forget to save?')
729 730 return
730 731 else:
731 732 self.shell.showtraceback()
@@ -1,1479 +1,1478 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Implementation of execution-related magic functions."""
3 3
4 4 # Copyright (c) IPython Development Team.
5 5 # Distributed under the terms of the Modified BSD License.
6 6
7 7
8 8 import ast
9 9 import bdb
10 10 import builtins as builtin_mod
11 11 import gc
12 12 import itertools
13 13 import os
14 14 import shlex
15 15 import sys
16 16 import time
17 17 import timeit
18 18 import math
19 19 import re
20 20 from pdb import Restart
21 21
22 22 # cProfile was added in Python2.5
23 23 try:
24 24 import cProfile as profile
25 25 import pstats
26 26 except ImportError:
27 27 # profile isn't bundled by default in Debian for license reasons
28 28 try:
29 29 import profile, pstats
30 30 except ImportError:
31 31 profile = pstats = None
32 32
33 33 from IPython.core import oinspect
34 34 from IPython.core import magic_arguments
35 35 from IPython.core import page
36 36 from IPython.core.error import UsageError
37 37 from IPython.core.macro import Macro
38 38 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
39 39 line_cell_magic, on_off, needs_local_scope,
40 40 no_var_expand)
41 41 from IPython.testing.skipdoctest import skip_doctest
42 42 from IPython.utils.contexts import preserve_keys
43 43 from IPython.utils.capture import capture_output
44 44 from IPython.utils.ipstruct import Struct
45 45 from IPython.utils.module_paths import find_mod
46 46 from IPython.utils.path import get_py_filename, shellglob
47 47 from IPython.utils.timing import clock, clock2
48 48 from warnings import warn
49 49 from logging import error
50 50 from io import StringIO
51 51
52 52
53 53 #-----------------------------------------------------------------------------
54 54 # Magic implementation classes
55 55 #-----------------------------------------------------------------------------
56 56
57 57
58 58 class TimeitResult(object):
59 59 """
60 60 Object returned by the timeit magic with info about the run.
61 61
62 62 Contains the following attributes :
63 63
64 64 loops: (int) number of loops done per measurement
65 65 repeat: (int) number of times the measurement has been repeated
66 66 best: (float) best execution time / number
67 67 all_runs: (list of float) execution time of each run (in s)
68 68 compile_time: (float) time of statement compilation (s)
69 69
70 70 """
71 71 def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
72 72 self.loops = loops
73 73 self.repeat = repeat
74 74 self.best = best
75 75 self.worst = worst
76 76 self.all_runs = all_runs
77 77 self.compile_time = compile_time
78 78 self._precision = precision
79 79 self.timings = [ dt / self.loops for dt in all_runs]
80 80
81 81 @property
82 82 def average(self):
83 83 return math.fsum(self.timings) / len(self.timings)
84 84
85 85 @property
86 86 def stdev(self):
87 87 mean = self.average
88 88 return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
89 89
90 90 def __str__(self):
91 91 pm = '+-'
92 92 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
93 93 try:
94 94 u'\xb1'.encode(sys.stdout.encoding)
95 95 pm = u'\xb1'
96 96 except:
97 97 pass
98 98 return (
99 99 u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
100 100 .format(
101 101 pm = pm,
102 102 runs = self.repeat,
103 103 loops = self.loops,
104 104 loop_plural = "" if self.loops == 1 else "s",
105 105 run_plural = "" if self.repeat == 1 else "s",
106 106 mean = _format_time(self.average, self._precision),
107 107 std = _format_time(self.stdev, self._precision))
108 108 )
109 109
110 110 def _repr_pretty_(self, p , cycle):
111 111 unic = self.__str__()
112 112 p.text(u'<TimeitResult : '+unic+u'>')
113 113
114 114
115 115 class TimeitTemplateFiller(ast.NodeTransformer):
116 116 """Fill in the AST template for timing execution.
117 117
118 118 This is quite closely tied to the template definition, which is in
119 119 :meth:`ExecutionMagics.timeit`.
120 120 """
121 121 def __init__(self, ast_setup, ast_stmt):
122 122 self.ast_setup = ast_setup
123 123 self.ast_stmt = ast_stmt
124 124
125 125 def visit_FunctionDef(self, node):
126 126 "Fill in the setup statement"
127 127 self.generic_visit(node)
128 128 if node.name == "inner":
129 129 node.body[:1] = self.ast_setup.body
130 130
131 131 return node
132 132
133 133 def visit_For(self, node):
134 134 "Fill in the statement to be timed"
135 135 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
136 136 node.body = self.ast_stmt.body
137 137 return node
138 138
139 139
140 140 class Timer(timeit.Timer):
141 141 """Timer class that explicitly uses self.inner
142 142
143 143 which is an undocumented implementation detail of CPython,
144 144 not shared by PyPy.
145 145 """
146 146 # Timer.timeit copied from CPython 3.4.2
147 147 def timeit(self, number=timeit.default_number):
148 148 """Time 'number' executions of the main statement.
149 149
150 150 To be precise, this executes the setup statement once, and
151 151 then returns the time it takes to execute the main statement
152 152 a number of times, as a float measured in seconds. The
153 153 argument is the number of times through the loop, defaulting
154 154 to one million. The main statement, the setup statement and
155 155 the timer function to be used are passed to the constructor.
156 156 """
157 157 it = itertools.repeat(None, number)
158 158 gcold = gc.isenabled()
159 159 gc.disable()
160 160 try:
161 161 timing = self.inner(it, self.timer)
162 162 finally:
163 163 if gcold:
164 164 gc.enable()
165 165 return timing
166 166
167 167
168 168 @magics_class
169 169 class ExecutionMagics(Magics):
170 170 """Magics related to code execution, debugging, profiling, etc.
171 171
172 172 """
173 173
174 174 def __init__(self, shell):
175 175 super(ExecutionMagics, self).__init__(shell)
176 176 if profile is None:
177 177 self.prun = self.profile_missing_notice
178 178 # Default execution function used to actually run user code.
179 179 self.default_runner = None
180 180
181 181 def profile_missing_notice(self, *args, **kwargs):
182 182 error("""\
183 183 The profile module could not be found. It has been removed from the standard
184 184 python packages because of its non-free license. To use profiling, install the
185 185 python-profiler package from non-free.""")
186 186
187 187 @skip_doctest
188 188 @no_var_expand
189 189 @line_cell_magic
190 190 def prun(self, parameter_s='', cell=None):
191 191
192 192 """Run a statement through the python code profiler.
193 193
194 194 Usage, in line mode:
195 195 %prun [options] statement
196 196
197 197 Usage, in cell mode:
198 198 %%prun [options] [statement]
199 199 code...
200 200 code...
201 201
202 202 In cell mode, the additional code lines are appended to the (possibly
203 203 empty) statement in the first line. Cell mode allows you to easily
204 204 profile multiline blocks without having to put them in a separate
205 205 function.
206 206
207 207 The given statement (which doesn't require quote marks) is run via the
208 208 python profiler in a manner similar to the profile.run() function.
209 209 Namespaces are internally managed to work correctly; profile.run
210 210 cannot be used in IPython because it makes certain assumptions about
211 211 namespaces which do not hold under IPython.
212 212
213 213 Options:
214 214
215 215 -l <limit>
216 216 you can place restrictions on what or how much of the
217 217 profile gets printed. The limit value can be:
218 218
219 219 * A string: only information for function names containing this string
220 220 is printed.
221 221
222 222 * An integer: only these many lines are printed.
223 223
224 224 * A float (between 0 and 1): this fraction of the report is printed
225 225 (for example, use a limit of 0.4 to see the topmost 40% only).
226 226
227 227 You can combine several limits with repeated use of the option. For
228 228 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
229 229 information about class constructors.
230 230
231 231 -r
232 232 return the pstats.Stats object generated by the profiling. This
233 233 object has all the information about the profile in it, and you can
234 234 later use it for further analysis or in other functions.
235 235
236 236 -s <key>
237 237 sort profile by given key. You can provide more than one key
238 238 by using the option several times: '-s key1 -s key2 -s key3...'. The
239 239 default sorting key is 'time'.
240 240
241 241 The following is copied verbatim from the profile documentation
242 242 referenced below:
243 243
244 244 When more than one key is provided, additional keys are used as
245 245 secondary criteria when the there is equality in all keys selected
246 246 before them.
247 247
248 248 Abbreviations can be used for any key names, as long as the
249 249 abbreviation is unambiguous. The following are the keys currently
250 250 defined:
251 251
252 252 ============ =====================
253 253 Valid Arg Meaning
254 254 ============ =====================
255 255 "calls" call count
256 256 "cumulative" cumulative time
257 257 "file" file name
258 258 "module" file name
259 259 "pcalls" primitive call count
260 260 "line" line number
261 261 "name" function name
262 262 "nfl" name/file/line
263 263 "stdname" standard name
264 264 "time" internal time
265 265 ============ =====================
266 266
267 267 Note that all sorts on statistics are in descending order (placing
268 268 most time consuming items first), where as name, file, and line number
269 269 searches are in ascending order (i.e., alphabetical). The subtle
270 270 distinction between "nfl" and "stdname" is that the standard name is a
271 271 sort of the name as printed, which means that the embedded line
272 272 numbers get compared in an odd way. For example, lines 3, 20, and 40
273 273 would (if the file names were the same) appear in the string order
274 274 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
275 275 line numbers. In fact, sort_stats("nfl") is the same as
276 276 sort_stats("name", "file", "line").
277 277
278 278 -T <filename>
279 279 save profile results as shown on screen to a text
280 280 file. The profile is still shown on screen.
281 281
282 282 -D <filename>
283 283 save (via dump_stats) profile statistics to given
284 284 filename. This data is in a format understood by the pstats module, and
285 285 is generated by a call to the dump_stats() method of profile
286 286 objects. The profile is still shown on screen.
287 287
288 288 -q
289 289 suppress output to the pager. Best used with -T and/or -D above.
290 290
291 291 If you want to run complete programs under the profiler's control, use
292 292 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
293 293 contains profiler specific options as described here.
294 294
295 295 You can read the complete documentation for the profile module with::
296 296
297 297 In [1]: import profile; profile.help()
298 298
299 299 .. versionchanged:: 7.3
300 300 User variables are no longer expanded,
301 301 the magic line is always left unmodified.
302 302
303 303 """
304 304 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
305 305 list_all=True, posix=False)
306 306 if cell is not None:
307 307 arg_str += '\n' + cell
308 308 arg_str = self.shell.transform_cell(arg_str)
309 309 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
310 310
311 311 def _run_with_profiler(self, code, opts, namespace):
312 312 """
313 313 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
314 314
315 315 Parameters
316 316 ----------
317 317 code : str
318 318 Code to be executed.
319 319 opts : Struct
320 320 Options parsed by `self.parse_options`.
321 321 namespace : dict
322 322 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
323 323
324 324 """
325 325
326 326 # Fill default values for unspecified options:
327 327 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
328 328
329 329 prof = profile.Profile()
330 330 try:
331 331 prof = prof.runctx(code, namespace, namespace)
332 332 sys_exit = ''
333 333 except SystemExit:
334 334 sys_exit = """*** SystemExit exception caught in code being profiled."""
335 335
336 336 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
337 337
338 338 lims = opts.l
339 339 if lims:
340 340 lims = [] # rebuild lims with ints/floats/strings
341 341 for lim in opts.l:
342 342 try:
343 343 lims.append(int(lim))
344 344 except ValueError:
345 345 try:
346 346 lims.append(float(lim))
347 347 except ValueError:
348 348 lims.append(lim)
349 349
350 350 # Trap output.
351 351 stdout_trap = StringIO()
352 352 stats_stream = stats.stream
353 353 try:
354 354 stats.stream = stdout_trap
355 355 stats.print_stats(*lims)
356 356 finally:
357 357 stats.stream = stats_stream
358 358
359 359 output = stdout_trap.getvalue()
360 360 output = output.rstrip()
361 361
362 362 if 'q' not in opts:
363 363 page.page(output)
364 364 print(sys_exit, end=' ')
365 365
366 366 dump_file = opts.D[0]
367 367 text_file = opts.T[0]
368 368 if dump_file:
369 369 prof.dump_stats(dump_file)
370 370 print('\n*** Profile stats marshalled to file',\
371 371 repr(dump_file)+'.',sys_exit)
372 372 if text_file:
373 pfile = open(text_file,'w')
374 pfile.write(output)
375 pfile.close()
373 with open(text_file, 'w') as pfile:
374 pfile.write(output)
376 375 print('\n*** Profile printout saved to text file',\
377 376 repr(text_file)+'.',sys_exit)
378 377
379 378 if 'r' in opts:
380 379 return stats
381 380 else:
382 381 return None
383 382
384 383 @line_magic
385 384 def pdb(self, parameter_s=''):
386 385 """Control the automatic calling of the pdb interactive debugger.
387 386
388 387 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
389 388 argument it works as a toggle.
390 389
391 390 When an exception is triggered, IPython can optionally call the
392 391 interactive pdb debugger after the traceback printout. %pdb toggles
393 392 this feature on and off.
394 393
395 394 The initial state of this feature is set in your configuration
396 395 file (the option is ``InteractiveShell.pdb``).
397 396
398 397 If you want to just activate the debugger AFTER an exception has fired,
399 398 without having to type '%pdb on' and rerunning your code, you can use
400 399 the %debug magic."""
401 400
402 401 par = parameter_s.strip().lower()
403 402
404 403 if par:
405 404 try:
406 405 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
407 406 except KeyError:
408 407 print ('Incorrect argument. Use on/1, off/0, '
409 408 'or nothing for a toggle.')
410 409 return
411 410 else:
412 411 # toggle
413 412 new_pdb = not self.shell.call_pdb
414 413
415 414 # set on the shell
416 415 self.shell.call_pdb = new_pdb
417 416 print('Automatic pdb calling has been turned',on_off(new_pdb))
418 417
419 418 @skip_doctest
420 419 @magic_arguments.magic_arguments()
421 420 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
422 421 help="""
423 422 Set break point at LINE in FILE.
424 423 """
425 424 )
426 425 @magic_arguments.argument('statement', nargs='*',
427 426 help="""
428 427 Code to run in debugger.
429 428 You can omit this in cell magic mode.
430 429 """
431 430 )
432 431 @no_var_expand
433 432 @line_cell_magic
434 433 def debug(self, line='', cell=None):
435 434 """Activate the interactive debugger.
436 435
437 436 This magic command support two ways of activating debugger.
438 437 One is to activate debugger before executing code. This way, you
439 438 can set a break point, to step through the code from the point.
440 439 You can use this mode by giving statements to execute and optionally
441 440 a breakpoint.
442 441
443 442 The other one is to activate debugger in post-mortem mode. You can
444 443 activate this mode simply running %debug without any argument.
445 444 If an exception has just occurred, this lets you inspect its stack
446 445 frames interactively. Note that this will always work only on the last
447 446 traceback that occurred, so you must call this quickly after an
448 447 exception that you wish to inspect has fired, because if another one
449 448 occurs, it clobbers the previous one.
450 449
451 450 If you want IPython to automatically do this on every exception, see
452 451 the %pdb magic for more details.
453 452
454 453 .. versionchanged:: 7.3
455 454 When running code, user variables are no longer expanded,
456 455 the magic line is always left unmodified.
457 456
458 457 """
459 458 args = magic_arguments.parse_argstring(self.debug, line)
460 459
461 460 if not (args.breakpoint or args.statement or cell):
462 461 self._debug_post_mortem()
463 462 else:
464 463 code = "\n".join(args.statement)
465 464 if cell:
466 465 code += "\n" + cell
467 466 self._debug_exec(code, args.breakpoint)
468 467
469 468 def _debug_post_mortem(self):
470 469 self.shell.debugger(force=True)
471 470
472 471 def _debug_exec(self, code, breakpoint):
473 472 if breakpoint:
474 473 (filename, bp_line) = breakpoint.rsplit(':', 1)
475 474 bp_line = int(bp_line)
476 475 else:
477 476 (filename, bp_line) = (None, None)
478 477 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
479 478
480 479 @line_magic
481 480 def tb(self, s):
482 481 """Print the last traceback.
483 482
484 483 Optionally, specify an exception reporting mode, tuning the
485 484 verbosity of the traceback. By default the currently-active exception
486 485 mode is used. See %xmode for changing exception reporting modes.
487 486
488 487 Valid modes: Plain, Context, Verbose, and Minimal.
489 488 """
490 489 interactive_tb = self.shell.InteractiveTB
491 490 if s:
492 491 # Switch exception reporting mode for this one call.
493 492 # Ensure it is switched back.
494 493 def xmode_switch_err(name):
495 494 warn('Error changing %s exception modes.\n%s' %
496 495 (name,sys.exc_info()[1]))
497 496
498 497 new_mode = s.strip().capitalize()
499 498 original_mode = interactive_tb.mode
500 499 try:
501 500 try:
502 501 interactive_tb.set_mode(mode=new_mode)
503 502 except Exception:
504 503 xmode_switch_err('user')
505 504 else:
506 505 self.shell.showtraceback()
507 506 finally:
508 507 interactive_tb.set_mode(mode=original_mode)
509 508 else:
510 509 self.shell.showtraceback()
511 510
512 511 @skip_doctest
513 512 @line_magic
514 513 def run(self, parameter_s='', runner=None,
515 514 file_finder=get_py_filename):
516 515 """Run the named file inside IPython as a program.
517 516
518 517 Usage::
519 518
520 519 %run [-n -i -e -G]
521 520 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
522 521 ( -m mod | file ) [args]
523 522
524 523 Parameters after the filename are passed as command-line arguments to
525 524 the program (put in sys.argv). Then, control returns to IPython's
526 525 prompt.
527 526
528 527 This is similar to running at a system prompt ``python file args``,
529 528 but with the advantage of giving you IPython's tracebacks, and of
530 529 loading all variables into your interactive namespace for further use
531 530 (unless -p is used, see below).
532 531
533 532 The file is executed in a namespace initially consisting only of
534 533 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
535 534 sees its environment as if it were being run as a stand-alone program
536 535 (except for sharing global objects such as previously imported
537 536 modules). But after execution, the IPython interactive namespace gets
538 537 updated with all variables defined in the program (except for __name__
539 538 and sys.argv). This allows for very convenient loading of code for
540 539 interactive work, while giving each program a 'clean sheet' to run in.
541 540
542 541 Arguments are expanded using shell-like glob match. Patterns
543 542 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
544 543 tilde '~' will be expanded into user's home directory. Unlike
545 544 real shells, quotation does not suppress expansions. Use
546 545 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
547 546 To completely disable these expansions, you can use -G flag.
548 547
549 548 On Windows systems, the use of single quotes `'` when specifying
550 549 a file is not supported. Use double quotes `"`.
551 550
552 551 Options:
553 552
554 553 -n
555 554 __name__ is NOT set to '__main__', but to the running file's name
556 555 without extension (as python does under import). This allows running
557 556 scripts and reloading the definitions in them without calling code
558 557 protected by an ``if __name__ == "__main__"`` clause.
559 558
560 559 -i
561 560 run the file in IPython's namespace instead of an empty one. This
562 561 is useful if you are experimenting with code written in a text editor
563 562 which depends on variables defined interactively.
564 563
565 564 -e
566 565 ignore sys.exit() calls or SystemExit exceptions in the script
567 566 being run. This is particularly useful if IPython is being used to
568 567 run unittests, which always exit with a sys.exit() call. In such
569 568 cases you are interested in the output of the test results, not in
570 569 seeing a traceback of the unittest module.
571 570
572 571 -t
573 572 print timing information at the end of the run. IPython will give
574 573 you an estimated CPU time consumption for your script, which under
575 574 Unix uses the resource module to avoid the wraparound problems of
576 575 time.clock(). Under Unix, an estimate of time spent on system tasks
577 576 is also given (for Windows platforms this is reported as 0.0).
578 577
579 578 If -t is given, an additional ``-N<N>`` option can be given, where <N>
580 579 must be an integer indicating how many times you want the script to
581 580 run. The final timing report will include total and per run results.
582 581
583 582 For example (testing the script uniq_stable.py)::
584 583
585 584 In [1]: run -t uniq_stable
586 585
587 586 IPython CPU timings (estimated):
588 587 User : 0.19597 s.
589 588 System: 0.0 s.
590 589
591 590 In [2]: run -t -N5 uniq_stable
592 591
593 592 IPython CPU timings (estimated):
594 593 Total runs performed: 5
595 594 Times : Total Per run
596 595 User : 0.910862 s, 0.1821724 s.
597 596 System: 0.0 s, 0.0 s.
598 597
599 598 -d
600 599 run your program under the control of pdb, the Python debugger.
601 600 This allows you to execute your program step by step, watch variables,
602 601 etc. Internally, what IPython does is similar to calling::
603 602
604 603 pdb.run('execfile("YOURFILENAME")')
605 604
606 605 with a breakpoint set on line 1 of your file. You can change the line
607 606 number for this automatic breakpoint to be <N> by using the -bN option
608 607 (where N must be an integer). For example::
609 608
610 609 %run -d -b40 myscript
611 610
612 611 will set the first breakpoint at line 40 in myscript.py. Note that
613 612 the first breakpoint must be set on a line which actually does
614 613 something (not a comment or docstring) for it to stop execution.
615 614
616 615 Or you can specify a breakpoint in a different file::
617 616
618 617 %run -d -b myotherfile.py:20 myscript
619 618
620 619 When the pdb debugger starts, you will see a (Pdb) prompt. You must
621 620 first enter 'c' (without quotes) to start execution up to the first
622 621 breakpoint.
623 622
624 623 Entering 'help' gives information about the use of the debugger. You
625 624 can easily see pdb's full documentation with "import pdb;pdb.help()"
626 625 at a prompt.
627 626
628 627 -p
629 628 run program under the control of the Python profiler module (which
630 629 prints a detailed report of execution times, function calls, etc).
631 630
632 631 You can pass other options after -p which affect the behavior of the
633 632 profiler itself. See the docs for %prun for details.
634 633
635 634 In this mode, the program's variables do NOT propagate back to the
636 635 IPython interactive namespace (because they remain in the namespace
637 636 where the profiler executes them).
638 637
639 638 Internally this triggers a call to %prun, see its documentation for
640 639 details on the options available specifically for profiling.
641 640
642 641 There is one special usage for which the text above doesn't apply:
643 642 if the filename ends with .ipy[nb], the file is run as ipython script,
644 643 just as if the commands were written on IPython prompt.
645 644
646 645 -m
647 646 specify module name to load instead of script path. Similar to
648 647 the -m option for the python interpreter. Use this option last if you
649 648 want to combine with other %run options. Unlike the python interpreter
650 649 only source modules are allowed no .pyc or .pyo files.
651 650 For example::
652 651
653 652 %run -m example
654 653
655 654 will run the example module.
656 655
657 656 -G
658 657 disable shell-like glob expansion of arguments.
659 658
660 659 """
661 660
662 661 # Logic to handle issue #3664
663 662 # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
664 663 if '-m' in parameter_s and '--' not in parameter_s:
665 664 argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
666 665 for idx, arg in enumerate(argv):
667 666 if arg and arg.startswith('-') and arg != '-':
668 667 if arg == '-m':
669 668 argv.insert(idx + 2, '--')
670 669 break
671 670 else:
672 671 # Positional arg, break
673 672 break
674 673 parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
675 674
676 675 # get arguments and set sys.argv for program to be run.
677 676 opts, arg_lst = self.parse_options(parameter_s,
678 677 'nidtN:b:pD:l:rs:T:em:G',
679 678 mode='list', list_all=1)
680 679 if "m" in opts:
681 680 modulename = opts["m"][0]
682 681 modpath = find_mod(modulename)
683 682 if modpath is None:
684 683 warn('%r is not a valid modulename on sys.path'%modulename)
685 684 return
686 685 arg_lst = [modpath] + arg_lst
687 686 try:
688 687 fpath = None # initialize to make sure fpath is in scope later
689 688 fpath = arg_lst[0]
690 689 filename = file_finder(fpath)
691 690 except IndexError:
692 691 warn('you must provide at least a filename.')
693 692 print('\n%run:\n', oinspect.getdoc(self.run))
694 693 return
695 694 except IOError as e:
696 695 try:
697 696 msg = str(e)
698 697 except UnicodeError:
699 698 msg = e.message
700 699 if os.name == 'nt' and re.match(r"^'.*'$",fpath):
701 700 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
702 701 error(msg)
703 702 return
704 703
705 704 if filename.lower().endswith(('.ipy', '.ipynb')):
706 705 with preserve_keys(self.shell.user_ns, '__file__'):
707 706 self.shell.user_ns['__file__'] = filename
708 707 self.shell.safe_execfile_ipy(filename)
709 708 return
710 709
711 710 # Control the response to exit() calls made by the script being run
712 711 exit_ignore = 'e' in opts
713 712
714 713 # Make sure that the running script gets a proper sys.argv as if it
715 714 # were run from a system shell.
716 715 save_argv = sys.argv # save it for later restoring
717 716
718 717 if 'G' in opts:
719 718 args = arg_lst[1:]
720 719 else:
721 720 # tilde and glob expansion
722 721 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
723 722
724 723 sys.argv = [filename] + args # put in the proper filename
725 724
726 725 if 'n' in opts:
727 726 name = os.path.splitext(os.path.basename(filename))[0]
728 727 else:
729 728 name = '__main__'
730 729
731 730 if 'i' in opts:
732 731 # Run in user's interactive namespace
733 732 prog_ns = self.shell.user_ns
734 733 __name__save = self.shell.user_ns['__name__']
735 734 prog_ns['__name__'] = name
736 735 main_mod = self.shell.user_module
737 736
738 737 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
739 738 # set the __file__ global in the script's namespace
740 739 # TK: Is this necessary in interactive mode?
741 740 prog_ns['__file__'] = filename
742 741 else:
743 742 # Run in a fresh, empty namespace
744 743
745 744 # The shell MUST hold a reference to prog_ns so after %run
746 745 # exits, the python deletion mechanism doesn't zero it out
747 746 # (leaving dangling references). See interactiveshell for details
748 747 main_mod = self.shell.new_main_mod(filename, name)
749 748 prog_ns = main_mod.__dict__
750 749
751 750 # pickle fix. See interactiveshell for an explanation. But we need to
752 751 # make sure that, if we overwrite __main__, we replace it at the end
753 752 main_mod_name = prog_ns['__name__']
754 753
755 754 if main_mod_name == '__main__':
756 755 restore_main = sys.modules['__main__']
757 756 else:
758 757 restore_main = False
759 758
760 759 # This needs to be undone at the end to prevent holding references to
761 760 # every single object ever created.
762 761 sys.modules[main_mod_name] = main_mod
763 762
764 763 if 'p' in opts or 'd' in opts:
765 764 if 'm' in opts:
766 765 code = 'run_module(modulename, prog_ns)'
767 766 code_ns = {
768 767 'run_module': self.shell.safe_run_module,
769 768 'prog_ns': prog_ns,
770 769 'modulename': modulename,
771 770 }
772 771 else:
773 772 if 'd' in opts:
774 773 # allow exceptions to raise in debug mode
775 774 code = 'execfile(filename, prog_ns, raise_exceptions=True)'
776 775 else:
777 776 code = 'execfile(filename, prog_ns)'
778 777 code_ns = {
779 778 'execfile': self.shell.safe_execfile,
780 779 'prog_ns': prog_ns,
781 780 'filename': get_py_filename(filename),
782 781 }
783 782
784 783 try:
785 784 stats = None
786 785 if 'p' in opts:
787 786 stats = self._run_with_profiler(code, opts, code_ns)
788 787 else:
789 788 if 'd' in opts:
790 789 bp_file, bp_line = parse_breakpoint(
791 790 opts.get('b', ['1'])[0], filename)
792 791 self._run_with_debugger(
793 792 code, code_ns, filename, bp_line, bp_file)
794 793 else:
795 794 if 'm' in opts:
796 795 def run():
797 796 self.shell.safe_run_module(modulename, prog_ns)
798 797 else:
799 798 if runner is None:
800 799 runner = self.default_runner
801 800 if runner is None:
802 801 runner = self.shell.safe_execfile
803 802
804 803 def run():
805 804 runner(filename, prog_ns, prog_ns,
806 805 exit_ignore=exit_ignore)
807 806
808 807 if 't' in opts:
809 808 # timed execution
810 809 try:
811 810 nruns = int(opts['N'][0])
812 811 if nruns < 1:
813 812 error('Number of runs must be >=1')
814 813 return
815 814 except (KeyError):
816 815 nruns = 1
817 816 self._run_with_timing(run, nruns)
818 817 else:
819 818 # regular execution
820 819 run()
821 820
822 821 if 'i' in opts:
823 822 self.shell.user_ns['__name__'] = __name__save
824 823 else:
825 824 # update IPython interactive namespace
826 825
827 826 # Some forms of read errors on the file may mean the
828 827 # __name__ key was never set; using pop we don't have to
829 828 # worry about a possible KeyError.
830 829 prog_ns.pop('__name__', None)
831 830
832 831 with preserve_keys(self.shell.user_ns, '__file__'):
833 832 self.shell.user_ns.update(prog_ns)
834 833 finally:
835 834 # It's a bit of a mystery why, but __builtins__ can change from
836 835 # being a module to becoming a dict missing some key data after
837 836 # %run. As best I can see, this is NOT something IPython is doing
838 837 # at all, and similar problems have been reported before:
839 838 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
840 839 # Since this seems to be done by the interpreter itself, the best
841 840 # we can do is to at least restore __builtins__ for the user on
842 841 # exit.
843 842 self.shell.user_ns['__builtins__'] = builtin_mod
844 843
845 844 # Ensure key global structures are restored
846 845 sys.argv = save_argv
847 846 if restore_main:
848 847 sys.modules['__main__'] = restore_main
849 848 else:
850 849 # Remove from sys.modules the reference to main_mod we'd
851 850 # added. Otherwise it will trap references to objects
852 851 # contained therein.
853 852 del sys.modules[main_mod_name]
854 853
855 854 return stats
856 855
857 856 def _run_with_debugger(self, code, code_ns, filename=None,
858 857 bp_line=None, bp_file=None):
859 858 """
860 859 Run `code` in debugger with a break point.
861 860
862 861 Parameters
863 862 ----------
864 863 code : str
865 864 Code to execute.
866 865 code_ns : dict
867 866 A namespace in which `code` is executed.
868 867 filename : str
869 868 `code` is ran as if it is in `filename`.
870 869 bp_line : int, optional
871 870 Line number of the break point.
872 871 bp_file : str, optional
873 872 Path to the file in which break point is specified.
874 873 `filename` is used if not given.
875 874
876 875 Raises
877 876 ------
878 877 UsageError
879 878 If the break point given by `bp_line` is not valid.
880 879
881 880 """
882 881 deb = self.shell.InteractiveTB.pdb
883 882 if not deb:
884 883 self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
885 884 deb = self.shell.InteractiveTB.pdb
886 885
887 886 # deb.checkline() fails if deb.curframe exists but is None; it can
888 887 # handle it not existing. https://github.com/ipython/ipython/issues/10028
889 888 if hasattr(deb, 'curframe'):
890 889 del deb.curframe
891 890
892 891 # reset Breakpoint state, which is moronically kept
893 892 # in a class
894 893 bdb.Breakpoint.next = 1
895 894 bdb.Breakpoint.bplist = {}
896 895 bdb.Breakpoint.bpbynumber = [None]
897 896 deb.clear_all_breaks()
898 897 if bp_line is not None:
899 898 # Set an initial breakpoint to stop execution
900 899 maxtries = 10
901 900 bp_file = bp_file or filename
902 901 checkline = deb.checkline(bp_file, bp_line)
903 902 if not checkline:
904 903 for bp in range(bp_line + 1, bp_line + maxtries + 1):
905 904 if deb.checkline(bp_file, bp):
906 905 break
907 906 else:
908 907 msg = ("\nI failed to find a valid line to set "
909 908 "a breakpoint\n"
910 909 "after trying up to line: %s.\n"
911 910 "Please set a valid breakpoint manually "
912 911 "with the -b option." % bp)
913 912 raise UsageError(msg)
914 913 # if we find a good linenumber, set the breakpoint
915 914 deb.do_break('%s:%s' % (bp_file, bp_line))
916 915
917 916 if filename:
918 917 # Mimic Pdb._runscript(...)
919 918 deb._wait_for_mainpyfile = True
920 919 deb.mainpyfile = deb.canonic(filename)
921 920
922 921 # Start file run
923 922 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
924 923 try:
925 924 if filename:
926 925 # save filename so it can be used by methods on the deb object
927 926 deb._exec_filename = filename
928 927 while True:
929 928 try:
930 929 deb.run(code, code_ns)
931 930 except Restart:
932 931 print("Restarting")
933 932 if filename:
934 933 deb._wait_for_mainpyfile = True
935 934 deb.mainpyfile = deb.canonic(filename)
936 935 continue
937 936 else:
938 937 break
939 938
940 939
941 940 except:
942 941 etype, value, tb = sys.exc_info()
943 942 # Skip three frames in the traceback: the %run one,
944 943 # one inside bdb.py, and the command-line typed by the
945 944 # user (run by exec in pdb itself).
946 945 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
947 946
948 947 @staticmethod
949 948 def _run_with_timing(run, nruns):
950 949 """
951 950 Run function `run` and print timing information.
952 951
953 952 Parameters
954 953 ----------
955 954 run : callable
956 955 Any callable object which takes no argument.
957 956 nruns : int
958 957 Number of times to execute `run`.
959 958
960 959 """
961 960 twall0 = time.perf_counter()
962 961 if nruns == 1:
963 962 t0 = clock2()
964 963 run()
965 964 t1 = clock2()
966 965 t_usr = t1[0] - t0[0]
967 966 t_sys = t1[1] - t0[1]
968 967 print("\nIPython CPU timings (estimated):")
969 968 print(" User : %10.2f s." % t_usr)
970 969 print(" System : %10.2f s." % t_sys)
971 970 else:
972 971 runs = range(nruns)
973 972 t0 = clock2()
974 973 for nr in runs:
975 974 run()
976 975 t1 = clock2()
977 976 t_usr = t1[0] - t0[0]
978 977 t_sys = t1[1] - t0[1]
979 978 print("\nIPython CPU timings (estimated):")
980 979 print("Total runs performed:", nruns)
981 980 print(" Times : %10s %10s" % ('Total', 'Per run'))
982 981 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
983 982 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
984 983 twall1 = time.perf_counter()
985 984 print("Wall time: %10.2f s." % (twall1 - twall0))
986 985
987 986 @skip_doctest
988 987 @no_var_expand
989 988 @line_cell_magic
990 989 @needs_local_scope
991 990 def timeit(self, line='', cell=None, local_ns=None):
992 991 """Time execution of a Python statement or expression
993 992
994 993 Usage, in line mode:
995 994 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
996 995 or in cell mode:
997 996 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
998 997 code
999 998 code...
1000 999
1001 1000 Time execution of a Python statement or expression using the timeit
1002 1001 module. This function can be used both as a line and cell magic:
1003 1002
1004 1003 - In line mode you can time a single-line statement (though multiple
1005 1004 ones can be chained with using semicolons).
1006 1005
1007 1006 - In cell mode, the statement in the first line is used as setup code
1008 1007 (executed but not timed) and the body of the cell is timed. The cell
1009 1008 body has access to any variables created in the setup code.
1010 1009
1011 1010 Options:
1012 1011 -n<N>: execute the given statement <N> times in a loop. If <N> is not
1013 1012 provided, <N> is determined so as to get sufficient accuracy.
1014 1013
1015 1014 -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
1016 1015 best result.
1017 1016 Default: 7
1018 1017
1019 1018 -t: use time.time to measure the time, which is the default on Unix.
1020 1019 This function measures wall time.
1021 1020
1022 1021 -c: use time.clock to measure the time, which is the default on
1023 1022 Windows and measures wall time. On Unix, resource.getrusage is used
1024 1023 instead and returns the CPU user time.
1025 1024
1026 1025 -p<P>: use a precision of <P> digits to display the timing result.
1027 1026 Default: 3
1028 1027
1029 1028 -q: Quiet, do not print result.
1030 1029
1031 1030 -o: return a TimeitResult that can be stored in a variable to inspect
1032 1031 the result in more details.
1033 1032
1034 1033 .. versionchanged:: 7.3
1035 1034 User variables are no longer expanded,
1036 1035 the magic line is always left unmodified.
1037 1036
1038 1037 Examples
1039 1038 --------
1040 1039 ::
1041 1040
1042 1041 In [1]: %timeit pass
1043 1042 8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
1044 1043
1045 1044 In [2]: u = None
1046 1045
1047 1046 In [3]: %timeit u is None
1048 1047 29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
1049 1048
1050 1049 In [4]: %timeit -r 4 u == None
1051 1050
1052 1051 In [5]: import time
1053 1052
1054 1053 In [6]: %timeit -n1 time.sleep(2)
1055 1054
1056 1055
1057 1056 The times reported by %timeit will be slightly higher than those
1058 1057 reported by the timeit.py script when variables are accessed. This is
1059 1058 due to the fact that %timeit executes the statement in the namespace
1060 1059 of the shell, compared with timeit.py, which uses a single setup
1061 1060 statement to import function or create variables. Generally, the bias
1062 1061 does not matter as long as results from timeit.py are not mixed with
1063 1062 those from %timeit."""
1064 1063
1065 1064 opts, stmt = self.parse_options(line,'n:r:tcp:qo',
1066 1065 posix=False, strict=False)
1067 1066 if stmt == "" and cell is None:
1068 1067 return
1069 1068
1070 1069 timefunc = timeit.default_timer
1071 1070 number = int(getattr(opts, "n", 0))
1072 1071 default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
1073 1072 repeat = int(getattr(opts, "r", default_repeat))
1074 1073 precision = int(getattr(opts, "p", 3))
1075 1074 quiet = 'q' in opts
1076 1075 return_result = 'o' in opts
1077 1076 if hasattr(opts, "t"):
1078 1077 timefunc = time.time
1079 1078 if hasattr(opts, "c"):
1080 1079 timefunc = clock
1081 1080
1082 1081 timer = Timer(timer=timefunc)
1083 1082 # this code has tight coupling to the inner workings of timeit.Timer,
1084 1083 # but is there a better way to achieve that the code stmt has access
1085 1084 # to the shell namespace?
1086 1085 transform = self.shell.transform_cell
1087 1086
1088 1087 if cell is None:
1089 1088 # called as line magic
1090 1089 ast_setup = self.shell.compile.ast_parse("pass")
1091 1090 ast_stmt = self.shell.compile.ast_parse(transform(stmt))
1092 1091 else:
1093 1092 ast_setup = self.shell.compile.ast_parse(transform(stmt))
1094 1093 ast_stmt = self.shell.compile.ast_parse(transform(cell))
1095 1094
1096 1095 ast_setup = self.shell.transform_ast(ast_setup)
1097 1096 ast_stmt = self.shell.transform_ast(ast_stmt)
1098 1097
1099 1098 # Check that these compile to valid Python code *outside* the timer func
1100 1099 # Invalid code may become valid when put inside the function & loop,
1101 1100 # which messes up error messages.
1102 1101 # https://github.com/ipython/ipython/issues/10636
1103 1102 self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
1104 1103 self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
1105 1104
1106 1105 # This codestring is taken from timeit.template - we fill it in as an
1107 1106 # AST, so that we can apply our AST transformations to the user code
1108 1107 # without affecting the timing code.
1109 1108 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
1110 1109 ' setup\n'
1111 1110 ' _t0 = _timer()\n'
1112 1111 ' for _i in _it:\n'
1113 1112 ' stmt\n'
1114 1113 ' _t1 = _timer()\n'
1115 1114 ' return _t1 - _t0\n')
1116 1115
1117 1116 timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
1118 1117 timeit_ast = ast.fix_missing_locations(timeit_ast)
1119 1118
1120 1119 # Track compilation time so it can be reported if too long
1121 1120 # Minimum time above which compilation time will be reported
1122 1121 tc_min = 0.1
1123 1122
1124 1123 t0 = clock()
1125 1124 code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
1126 1125 tc = clock()-t0
1127 1126
1128 1127 ns = {}
1129 1128 glob = self.shell.user_ns
1130 1129 # handles global vars with same name as local vars. We store them in conflict_globs.
1131 1130 if local_ns is not None:
1132 1131 conflict_globs = {}
1133 1132 for var_name, var_val in glob.items():
1134 1133 if var_name in local_ns:
1135 1134 conflict_globs[var_name] = var_val
1136 1135 glob.update(local_ns)
1137 1136
1138 1137 exec(code, glob, ns)
1139 1138 timer.inner = ns["inner"]
1140 1139
1141 1140 # This is used to check if there is a huge difference between the
1142 1141 # best and worst timings.
1143 1142 # Issue: https://github.com/ipython/ipython/issues/6471
1144 1143 if number == 0:
1145 1144 # determine number so that 0.2 <= total time < 2.0
1146 1145 for index in range(0, 10):
1147 1146 number = 10 ** index
1148 1147 time_number = timer.timeit(number)
1149 1148 if time_number >= 0.2:
1150 1149 break
1151 1150
1152 1151 all_runs = timer.repeat(repeat, number)
1153 1152 best = min(all_runs) / number
1154 1153 worst = max(all_runs) / number
1155 1154 timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
1156 1155
1157 1156 # Restore global vars from conflict_globs
1158 1157 if local_ns is not None:
1159 1158 if len(conflict_globs) > 0:
1160 1159 glob.update(conflict_globs)
1161 1160
1162 1161 if not quiet :
1163 1162 # Check best timing is greater than zero to avoid a
1164 1163 # ZeroDivisionError.
1165 1164 # In cases where the slowest timing is lesser than a micosecond
1166 1165 # we assume that it does not really matter if the fastest
1167 1166 # timing is 4 times faster than the slowest timing or not.
1168 1167 if worst > 4 * best and best > 0 and worst > 1e-6:
1169 1168 print("The slowest run took %0.2f times longer than the "
1170 1169 "fastest. This could mean that an intermediate result "
1171 1170 "is being cached." % (worst / best))
1172 1171
1173 1172 print( timeit_result )
1174 1173
1175 1174 if tc > tc_min:
1176 1175 print("Compiler time: %.2f s" % tc)
1177 1176 if return_result:
1178 1177 return timeit_result
1179 1178
1180 1179 @skip_doctest
1181 1180 @no_var_expand
1182 1181 @needs_local_scope
1183 1182 @line_cell_magic
1184 1183 def time(self,line='', cell=None, local_ns=None):
1185 1184 """Time execution of a Python statement or expression.
1186 1185
1187 1186 The CPU and wall clock times are printed, and the value of the
1188 1187 expression (if any) is returned. Note that under Win32, system time
1189 1188 is always reported as 0, since it can not be measured.
1190 1189
1191 1190 This function can be used both as a line and cell magic:
1192 1191
1193 1192 - In line mode you can time a single-line statement (though multiple
1194 1193 ones can be chained with using semicolons).
1195 1194
1196 1195 - In cell mode, you can time the cell body (a directly
1197 1196 following statement raises an error).
1198 1197
1199 1198 This function provides very basic timing functionality. Use the timeit
1200 1199 magic for more control over the measurement.
1201 1200
1202 1201 .. versionchanged:: 7.3
1203 1202 User variables are no longer expanded,
1204 1203 the magic line is always left unmodified.
1205 1204
1206 1205 Examples
1207 1206 --------
1208 1207 ::
1209 1208
1210 1209 In [1]: %time 2**128
1211 1210 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1212 1211 Wall time: 0.00
1213 1212 Out[1]: 340282366920938463463374607431768211456L
1214 1213
1215 1214 In [2]: n = 1000000
1216 1215
1217 1216 In [3]: %time sum(range(n))
1218 1217 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1219 1218 Wall time: 1.37
1220 1219 Out[3]: 499999500000L
1221 1220
1222 1221 In [4]: %time print 'hello world'
1223 1222 hello world
1224 1223 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1225 1224 Wall time: 0.00
1226 1225
1227 1226 Note that the time needed by Python to compile the given expression
1228 1227 will be reported if it is more than 0.1s. In this example, the
1229 1228 actual exponentiation is done by Python at compilation time, so while
1230 1229 the expression can take a noticeable amount of time to compute, that
1231 1230 time is purely due to the compilation:
1232 1231
1233 1232 In [5]: %time 3**9999;
1234 1233 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1235 1234 Wall time: 0.00 s
1236 1235
1237 1236 In [6]: %time 3**999999;
1238 1237 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1239 1238 Wall time: 0.00 s
1240 1239 Compiler : 0.78 s
1241 1240 """
1242 1241
1243 1242 # fail immediately if the given expression can't be compiled
1244 1243
1245 1244 if line and cell:
1246 1245 raise UsageError("Can't use statement directly after '%%time'!")
1247 1246
1248 1247 if cell:
1249 1248 expr = self.shell.transform_cell(cell)
1250 1249 else:
1251 1250 expr = self.shell.transform_cell(line)
1252 1251
1253 1252 # Minimum time above which parse time will be reported
1254 1253 tp_min = 0.1
1255 1254
1256 1255 t0 = clock()
1257 1256 expr_ast = self.shell.compile.ast_parse(expr)
1258 1257 tp = clock()-t0
1259 1258
1260 1259 # Apply AST transformations
1261 1260 expr_ast = self.shell.transform_ast(expr_ast)
1262 1261
1263 1262 # Minimum time above which compilation time will be reported
1264 1263 tc_min = 0.1
1265 1264
1266 1265 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1267 1266 mode = 'eval'
1268 1267 source = '<timed eval>'
1269 1268 expr_ast = ast.Expression(expr_ast.body[0].value)
1270 1269 else:
1271 1270 mode = 'exec'
1272 1271 source = '<timed exec>'
1273 1272 t0 = clock()
1274 1273 code = self.shell.compile(expr_ast, source, mode)
1275 1274 tc = clock()-t0
1276 1275
1277 1276 # skew measurement as little as possible
1278 1277 glob = self.shell.user_ns
1279 1278 wtime = time.time
1280 1279 # time execution
1281 1280 wall_st = wtime()
1282 1281 if mode=='eval':
1283 1282 st = clock2()
1284 1283 try:
1285 1284 out = eval(code, glob, local_ns)
1286 1285 except:
1287 1286 self.shell.showtraceback()
1288 1287 return
1289 1288 end = clock2()
1290 1289 else:
1291 1290 st = clock2()
1292 1291 try:
1293 1292 exec(code, glob, local_ns)
1294 1293 except:
1295 1294 self.shell.showtraceback()
1296 1295 return
1297 1296 end = clock2()
1298 1297 out = None
1299 1298 wall_end = wtime()
1300 1299 # Compute actual times and report
1301 1300 wall_time = wall_end-wall_st
1302 1301 cpu_user = end[0]-st[0]
1303 1302 cpu_sys = end[1]-st[1]
1304 1303 cpu_tot = cpu_user+cpu_sys
1305 1304 # On windows cpu_sys is always zero, so no new information to the next print
1306 1305 if sys.platform != 'win32':
1307 1306 print("CPU times: user %s, sys: %s, total: %s" % \
1308 1307 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
1309 1308 print("Wall time: %s" % _format_time(wall_time))
1310 1309 if tc > tc_min:
1311 1310 print("Compiler : %s" % _format_time(tc))
1312 1311 if tp > tp_min:
1313 1312 print("Parser : %s" % _format_time(tp))
1314 1313 return out
1315 1314
1316 1315 @skip_doctest
1317 1316 @line_magic
1318 1317 def macro(self, parameter_s=''):
1319 1318 """Define a macro for future re-execution. It accepts ranges of history,
1320 1319 filenames or string objects.
1321 1320
1322 1321 Usage:\\
1323 1322 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1324 1323
1325 1324 Options:
1326 1325
1327 1326 -r: use 'raw' input. By default, the 'processed' history is used,
1328 1327 so that magics are loaded in their transformed version to valid
1329 1328 Python. If this option is given, the raw input as typed at the
1330 1329 command line is used instead.
1331 1330
1332 1331 -q: quiet macro definition. By default, a tag line is printed
1333 1332 to indicate the macro has been created, and then the contents of
1334 1333 the macro are printed. If this option is given, then no printout
1335 1334 is produced once the macro is created.
1336 1335
1337 1336 This will define a global variable called `name` which is a string
1338 1337 made of joining the slices and lines you specify (n1,n2,... numbers
1339 1338 above) from your input history into a single string. This variable
1340 1339 acts like an automatic function which re-executes those lines as if
1341 1340 you had typed them. You just type 'name' at the prompt and the code
1342 1341 executes.
1343 1342
1344 1343 The syntax for indicating input ranges is described in %history.
1345 1344
1346 1345 Note: as a 'hidden' feature, you can also use traditional python slice
1347 1346 notation, where N:M means numbers N through M-1.
1348 1347
1349 1348 For example, if your history contains (print using %hist -n )::
1350 1349
1351 1350 44: x=1
1352 1351 45: y=3
1353 1352 46: z=x+y
1354 1353 47: print x
1355 1354 48: a=5
1356 1355 49: print 'x',x,'y',y
1357 1356
1358 1357 you can create a macro with lines 44 through 47 (included) and line 49
1359 1358 called my_macro with::
1360 1359
1361 1360 In [55]: %macro my_macro 44-47 49
1362 1361
1363 1362 Now, typing `my_macro` (without quotes) will re-execute all this code
1364 1363 in one pass.
1365 1364
1366 1365 You don't need to give the line-numbers in order, and any given line
1367 1366 number can appear multiple times. You can assemble macros with any
1368 1367 lines from your input history in any order.
1369 1368
1370 1369 The macro is a simple object which holds its value in an attribute,
1371 1370 but IPython's display system checks for macros and executes them as
1372 1371 code instead of printing them when you type their name.
1373 1372
1374 1373 You can view a macro's contents by explicitly printing it with::
1375 1374
1376 1375 print macro_name
1377 1376
1378 1377 """
1379 1378 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1380 1379 if not args: # List existing macros
1381 1380 return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
1382 1381 if len(args) == 1:
1383 1382 raise UsageError(
1384 1383 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1385 1384 name, codefrom = args[0], " ".join(args[1:])
1386 1385
1387 1386 #print 'rng',ranges # dbg
1388 1387 try:
1389 1388 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1390 1389 except (ValueError, TypeError) as e:
1391 1390 print(e.args[0])
1392 1391 return
1393 1392 macro = Macro(lines)
1394 1393 self.shell.define_macro(name, macro)
1395 1394 if not ( 'q' in opts) :
1396 1395 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1397 1396 print('=== Macro contents: ===')
1398 1397 print(macro, end=' ')
1399 1398
1400 1399 @magic_arguments.magic_arguments()
1401 1400 @magic_arguments.argument('output', type=str, default='', nargs='?',
1402 1401 help="""The name of the variable in which to store output.
1403 1402 This is a utils.io.CapturedIO object with stdout/err attributes
1404 1403 for the text of the captured output.
1405 1404
1406 1405 CapturedOutput also has a show() method for displaying the output,
1407 1406 and __call__ as well, so you can use that to quickly display the
1408 1407 output.
1409 1408
1410 1409 If unspecified, captured output is discarded.
1411 1410 """
1412 1411 )
1413 1412 @magic_arguments.argument('--no-stderr', action="store_true",
1414 1413 help="""Don't capture stderr."""
1415 1414 )
1416 1415 @magic_arguments.argument('--no-stdout', action="store_true",
1417 1416 help="""Don't capture stdout."""
1418 1417 )
1419 1418 @magic_arguments.argument('--no-display', action="store_true",
1420 1419 help="""Don't capture IPython's rich display."""
1421 1420 )
1422 1421 @cell_magic
1423 1422 def capture(self, line, cell):
1424 1423 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1425 1424 args = magic_arguments.parse_argstring(self.capture, line)
1426 1425 out = not args.no_stdout
1427 1426 err = not args.no_stderr
1428 1427 disp = not args.no_display
1429 1428 with capture_output(out, err, disp) as io:
1430 1429 self.shell.run_cell(cell)
1431 1430 if args.output:
1432 1431 self.shell.user_ns[args.output] = io
1433 1432
1434 1433 def parse_breakpoint(text, current_file):
1435 1434 '''Returns (file, line) for file:line and (current_file, line) for line'''
1436 1435 colon = text.find(':')
1437 1436 if colon == -1:
1438 1437 return current_file, int(text)
1439 1438 else:
1440 1439 return text[:colon], int(text[colon+1:])
1441 1440
1442 1441 def _format_time(timespan, precision=3):
1443 1442 """Formats the timespan in a human readable form"""
1444 1443
1445 1444 if timespan >= 60.0:
1446 1445 # we have more than a minute, format that in a human readable form
1447 1446 # Idea from http://snipplr.com/view/5713/
1448 1447 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1449 1448 time = []
1450 1449 leftover = timespan
1451 1450 for suffix, length in parts:
1452 1451 value = int(leftover / length)
1453 1452 if value > 0:
1454 1453 leftover = leftover % length
1455 1454 time.append(u'%s%s' % (str(value), suffix))
1456 1455 if leftover < 1:
1457 1456 break
1458 1457 return " ".join(time)
1459 1458
1460 1459
1461 1460 # Unfortunately the unicode 'micro' symbol can cause problems in
1462 1461 # certain terminals.
1463 1462 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1464 1463 # Try to prevent crashes by being more secure than it needs to
1465 1464 # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
1466 1465 units = [u"s", u"ms",u'us',"ns"] # the save value
1467 1466 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1468 1467 try:
1469 1468 u'\xb5'.encode(sys.stdout.encoding)
1470 1469 units = [u"s", u"ms",u'\xb5s',"ns"]
1471 1470 except:
1472 1471 pass
1473 1472 scaling = [1, 1e3, 1e6, 1e9]
1474 1473
1475 1474 if timespan > 0.0:
1476 1475 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1477 1476 else:
1478 1477 order = 3
1479 1478 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
@@ -1,103 +1,104 b''
1 1 """Implementation of packaging-related magic functions.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (c) 2018 The IPython Development Team.
5 5 #
6 6 # Distributed under the terms of the Modified BSD License.
7 7 #
8 8 # The full license is in the file COPYING.txt, distributed with this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 import os
12 12 import re
13 13 import shlex
14 14 import sys
15 15 from subprocess import Popen, PIPE
16 16
17 17 from IPython.core.magic import Magics, magics_class, line_magic
18 18
19 19
20 20 def _is_conda_environment():
21 21 """Return True if the current Python executable is in a conda env"""
22 22 # TODO: does this need to change on windows?
23 23 conda_history = os.path.join(sys.prefix, 'conda-meta', 'history')
24 24 return os.path.exists(conda_history)
25 25
26 26
27 27 def _get_conda_executable():
28 28 """Find the path to the conda executable"""
29 29 # Check if there is a conda executable in the same directory as the Python executable.
30 30 # This is the case within conda's root environment.
31 31 conda = os.path.join(os.path.dirname(sys.executable), 'conda')
32 32 if os.path.isfile(conda):
33 33 return conda
34 34
35 35 # Otherwise, attempt to extract the executable from conda history.
36 36 # This applies in any conda environment.
37 37 R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]")
38 for line in open(os.path.join(sys.prefix, 'conda-meta', 'history')):
39 match = R.match(line)
40 if match:
41 return match.groupdict()['command']
38 with open(os.path.join(sys.prefix, 'conda-meta', 'history')) as f:
39 for line in f:
40 match = R.match(line)
41 if match:
42 return match.groupdict()['command']
42 43
43 44 # Fallback: assume conda is available on the system path.
44 45 return "conda"
45 46
46 47
47 48 CONDA_COMMANDS_REQUIRING_PREFIX = {
48 49 'install', 'list', 'remove', 'uninstall', 'update', 'upgrade',
49 50 }
50 51 CONDA_COMMANDS_REQUIRING_YES = {
51 52 'install', 'remove', 'uninstall', 'update', 'upgrade',
52 53 }
53 54 CONDA_ENV_FLAGS = {'-p', '--prefix', '-n', '--name'}
54 55 CONDA_YES_FLAGS = {'-y', '--y'}
55 56
56 57
57 58 @magics_class
58 59 class PackagingMagics(Magics):
59 60 """Magics related to packaging & installation"""
60 61
61 62 @line_magic
62 63 def pip(self, line):
63 64 """Run the pip package manager within the current kernel.
64 65
65 66 Usage:
66 67 %pip install [pkgs]
67 68 """
68 69 self.shell.system(' '.join([sys.executable, '-m', 'pip', line]))
69 70 print("Note: you may need to restart the kernel to use updated packages.")
70 71
71 72 @line_magic
72 73 def conda(self, line):
73 74 """Run the conda package manager within the current kernel.
74 75
75 76 Usage:
76 77 %conda install [pkgs]
77 78 """
78 79 if not _is_conda_environment():
79 80 raise ValueError("The python kernel does not appear to be a conda environment. "
80 81 "Please use ``%pip install`` instead.")
81 82
82 83 conda = _get_conda_executable()
83 84 args = shlex.split(line)
84 85 command = args[0]
85 86 args = args[1:]
86 87 extra_args = []
87 88
88 89 # When the subprocess does not allow us to respond "yes" during the installation,
89 90 # we need to insert --yes in the argument list for some commands
90 91 stdin_disabled = getattr(self.shell, 'kernel', None) is not None
91 92 needs_yes = command in CONDA_COMMANDS_REQUIRING_YES
92 93 has_yes = set(args).intersection(CONDA_YES_FLAGS)
93 94 if stdin_disabled and needs_yes and not has_yes:
94 95 extra_args.append("--yes")
95 96
96 97 # Add --prefix to point conda installation to the current environment
97 98 needs_prefix = command in CONDA_COMMANDS_REQUIRING_PREFIX
98 99 has_prefix = set(args).intersection(CONDA_ENV_FLAGS)
99 100 if needs_prefix and not has_prefix:
100 101 extra_args.extend(["--prefix", sys.prefix])
101 102
102 103 self.shell.system(' '.join([conda, command] + extra_args + args))
103 104 print("\nNote: you may need to restart the kernel to use updated packages.") No newline at end of file
@@ -1,226 +1,225 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 %store magic for lightweight persistence.
4 4
5 5 Stores variables, aliases and macros in IPython's database.
6 6
7 7 To automatically restore stored variables at startup, add this to your
8 8 :file:`ipython_config.py` file::
9 9
10 10 c.StoreMagics.autorestore = True
11 11 """
12 12
13 13 # Copyright (c) IPython Development Team.
14 14 # Distributed under the terms of the Modified BSD License.
15 15
16 16 import inspect, os, sys, textwrap
17 17
18 18 from IPython.core.error import UsageError
19 19 from IPython.core.magic import Magics, magics_class, line_magic
20 20 from traitlets import Bool
21 21
22 22
23 23 def restore_aliases(ip):
24 24 staliases = ip.db.get('stored_aliases', {})
25 25 for k,v in staliases.items():
26 26 #print "restore alias",k,v # dbg
27 27 #self.alias_table[k] = v
28 28 ip.alias_manager.define_alias(k,v)
29 29
30 30
31 31 def refresh_variables(ip):
32 32 db = ip.db
33 33 for key in db.keys('autorestore/*'):
34 34 # strip autorestore
35 35 justkey = os.path.basename(key)
36 36 try:
37 37 obj = db[key]
38 38 except KeyError:
39 39 print("Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey)
40 40 print("The error was:", sys.exc_info()[0])
41 41 else:
42 42 #print "restored",justkey,"=",obj #dbg
43 43 ip.user_ns[justkey] = obj
44 44
45 45
46 46 def restore_dhist(ip):
47 47 ip.user_ns['_dh'] = ip.db.get('dhist',[])
48 48
49 49
50 50 def restore_data(ip):
51 51 refresh_variables(ip)
52 52 restore_aliases(ip)
53 53 restore_dhist(ip)
54 54
55 55
56 56 @magics_class
57 57 class StoreMagics(Magics):
58 58 """Lightweight persistence for python variables.
59 59
60 60 Provides the %store magic."""
61 61
62 62 autorestore = Bool(False, help=
63 63 """If True, any %store-d variables will be automatically restored
64 64 when IPython starts.
65 65 """
66 66 ).tag(config=True)
67 67
68 68 def __init__(self, shell):
69 69 super(StoreMagics, self).__init__(shell=shell)
70 70 self.shell.configurables.append(self)
71 71 if self.autorestore:
72 72 restore_data(self.shell)
73 73
74 74 @line_magic
75 75 def store(self, parameter_s=''):
76 76 """Lightweight persistence for python variables.
77 77
78 78 Example::
79 79
80 80 In [1]: l = ['hello',10,'world']
81 81 In [2]: %store l
82 82 In [3]: exit
83 83
84 84 (IPython session is closed and started again...)
85 85
86 86 ville@badger:~$ ipython
87 87 In [1]: l
88 88 NameError: name 'l' is not defined
89 89 In [2]: %store -r
90 90 In [3]: l
91 91 Out[3]: ['hello', 10, 'world']
92 92
93 93 Usage:
94 94
95 95 * ``%store`` - Show list of all variables and their current
96 96 values
97 97 * ``%store spam`` - Store the *current* value of the variable spam
98 98 to disk
99 99 * ``%store -d spam`` - Remove the variable and its value from storage
100 100 * ``%store -z`` - Remove all variables from storage
101 101 * ``%store -r`` - Refresh all variables from store (overwrite
102 102 current vals)
103 103 * ``%store -r spam bar`` - Refresh specified variables from store
104 104 (delete current val)
105 105 * ``%store foo >a.txt`` - Store value of foo to new file a.txt
106 106 * ``%store foo >>a.txt`` - Append value of foo to file a.txt
107 107
108 108 It should be noted that if you change the value of a variable, you
109 109 need to %store it again if you want to persist the new value.
110 110
111 111 Note also that the variables will need to be pickleable; most basic
112 112 python types can be safely %store'd.
113 113
114 114 Also aliases can be %store'd across sessions.
115 115 """
116 116
117 117 opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
118 118 args = argsl.split(None,1)
119 119 ip = self.shell
120 120 db = ip.db
121 121 # delete
122 122 if 'd' in opts:
123 123 try:
124 124 todel = args[0]
125 125 except IndexError:
126 126 raise UsageError('You must provide the variable to forget')
127 127 else:
128 128 try:
129 129 del db['autorestore/' + todel]
130 130 except:
131 131 raise UsageError("Can't delete variable '%s'" % todel)
132 132 # reset
133 133 elif 'z' in opts:
134 134 for k in db.keys('autorestore/*'):
135 135 del db[k]
136 136
137 137 elif 'r' in opts:
138 138 if args:
139 139 for arg in args:
140 140 try:
141 141 obj = db['autorestore/' + arg]
142 142 except KeyError:
143 143 print("no stored variable %s" % arg)
144 144 else:
145 145 ip.user_ns[arg] = obj
146 146 else:
147 147 restore_data(ip)
148 148
149 149 # run without arguments -> list variables & values
150 150 elif not args:
151 151 vars = db.keys('autorestore/*')
152 152 vars.sort()
153 153 if vars:
154 154 size = max(map(len, vars))
155 155 else:
156 156 size = 0
157 157
158 158 print('Stored variables and their in-db values:')
159 159 fmt = '%-'+str(size)+'s -> %s'
160 160 get = db.get
161 161 for var in vars:
162 162 justkey = os.path.basename(var)
163 163 # print 30 first characters from every var
164 164 print(fmt % (justkey, repr(get(var, '<unavailable>'))[:50]))
165 165
166 166 # default action - store the variable
167 167 else:
168 168 # %store foo >file.txt or >>file.txt
169 169 if len(args) > 1 and args[1].startswith('>'):
170 170 fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
171 171 if args[1].startswith('>>'):
172 172 fil = open(fnam, 'a')
173 173 else:
174 174 fil = open(fnam, 'w')
175 obj = ip.ev(args[0])
176 print("Writing '%s' (%s) to file '%s'." % (args[0],
177 obj.__class__.__name__, fnam))
178
179
180 if not isinstance (obj, str):
181 from pprint import pprint
182 pprint(obj, fil)
183 else:
184 fil.write(obj)
185 if not obj.endswith('\n'):
186 fil.write('\n')
175 with fil:
176 obj = ip.ev(args[0])
177 print("Writing '%s' (%s) to file '%s'." % (args[0],
178 obj.__class__.__name__, fnam))
179
180 if not isinstance (obj, str):
181 from pprint import pprint
182 pprint(obj, fil)
183 else:
184 fil.write(obj)
185 if not obj.endswith('\n'):
186 fil.write('\n')
187 187
188 fil.close()
189 188 return
190 189
191 190 # %store foo
192 191 try:
193 192 obj = ip.user_ns[args[0]]
194 193 except KeyError:
195 194 # it might be an alias
196 195 name = args[0]
197 196 try:
198 197 cmd = ip.alias_manager.retrieve_alias(name)
199 198 except ValueError:
200 199 raise UsageError("Unknown variable '%s'" % name)
201 200
202 201 staliases = db.get('stored_aliases',{})
203 202 staliases[name] = cmd
204 203 db['stored_aliases'] = staliases
205 204 print("Alias stored: %s (%s)" % (name, cmd))
206 205 return
207 206
208 207 else:
209 208 modname = getattr(inspect.getmodule(obj), '__name__', '')
210 209 if modname == '__main__':
211 210 print(textwrap.dedent("""\
212 211 Warning:%s is %s
213 212 Proper storage of interactively declared classes (or instances
214 213 of those classes) is not possible! Only instances
215 214 of classes in real modules on file system can be %%store'd.
216 215 """ % (args[0], obj) ))
217 216 return
218 217 #pickled = pickle.dumps(obj)
219 218 db[ 'autorestore/' + args[0] ] = obj
220 219 print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__))
221 220
222 221
223 222 def load_ipython_extension(ip):
224 223 """Load the extension in IPython."""
225 224 ip.register_magics(StoreMagics)
226 225
@@ -1,404 +1,398 b''
1 1 """Tests for autoreload extension.
2 2 """
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (c) 2012 IPython Development Team.
5 5 #
6 6 # Distributed under the terms of the Modified BSD License.
7 7 #
8 8 # The full license is in the file COPYING.txt, distributed with this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Imports
13 13 #-----------------------------------------------------------------------------
14 14
15 15 import os
16 16 import sys
17 17 import tempfile
18 18 import textwrap
19 19 import shutil
20 20 import random
21 21 import time
22 22 from io import StringIO
23 23
24 24 import nose.tools as nt
25 25 import IPython.testing.tools as tt
26 26
27 27 from IPython.testing.decorators import skipif
28 28
29 29 from IPython.extensions.autoreload import AutoreloadMagics
30 30 from IPython.core.events import EventManager, pre_run_cell
31 31
32 32 #-----------------------------------------------------------------------------
33 33 # Test fixture
34 34 #-----------------------------------------------------------------------------
35 35
36 36 noop = lambda *a, **kw: None
37 37
38 38 class FakeShell(object):
39 39
40 40 def __init__(self):
41 41 self.ns = {}
42 42 self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
43 43 self.auto_magics = AutoreloadMagics(shell=self)
44 44 self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
45 45
46 46 register_magics = set_hook = noop
47 47
48 48 def run_code(self, code):
49 49 self.events.trigger('pre_run_cell')
50 50 exec(code, self.ns)
51 51 self.auto_magics.post_execute_hook()
52 52
53 53 def push(self, items):
54 54 self.ns.update(items)
55 55
56 56 def magic_autoreload(self, parameter):
57 57 self.auto_magics.autoreload(parameter)
58 58
59 59 def magic_aimport(self, parameter, stream=None):
60 60 self.auto_magics.aimport(parameter, stream=stream)
61 61 self.auto_magics.post_execute_hook()
62 62
63 63
64 64 class Fixture(object):
65 65 """Fixture for creating test module files"""
66 66
67 67 test_dir = None
68 68 old_sys_path = None
69 69 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
70 70
71 71 def setUp(self):
72 72 self.test_dir = tempfile.mkdtemp()
73 73 self.old_sys_path = list(sys.path)
74 74 sys.path.insert(0, self.test_dir)
75 75 self.shell = FakeShell()
76 76
77 77 def tearDown(self):
78 78 shutil.rmtree(self.test_dir)
79 79 sys.path = self.old_sys_path
80 80
81 81 self.test_dir = None
82 82 self.old_sys_path = None
83 83 self.shell = None
84 84
85 85 def get_module(self):
86 86 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
87 87 if module_name in sys.modules:
88 88 del sys.modules[module_name]
89 89 file_name = os.path.join(self.test_dir, module_name + ".py")
90 90 return module_name, file_name
91 91
92 92 def write_file(self, filename, content):
93 93 """
94 94 Write a file, and force a timestamp difference of at least one second
95 95
96 96 Notes
97 97 -----
98 98 Python's .pyc files record the timestamp of their compilation
99 99 with a time resolution of one second.
100 100
101 101 Therefore, we need to force a timestamp difference between .py
102 102 and .pyc, without having the .py file be timestamped in the
103 103 future, and without changing the timestamp of the .pyc file
104 104 (because that is stored in the file). The only reliable way
105 105 to achieve this seems to be to sleep.
106 106 """
107 107
108 108 # Sleep one second + eps
109 109 time.sleep(1.05)
110 110
111 111 # Write
112 f = open(filename, 'w')
113 try:
112 with open(filename, 'w') as f:
114 113 f.write(content)
115 finally:
116 f.close()
117 114
118 115 def new_module(self, code):
119 116 mod_name, mod_fn = self.get_module()
120 f = open(mod_fn, 'w')
121 try:
117 with open(mod_fn, 'w') as f:
122 118 f.write(code)
123 finally:
124 f.close()
125 119 return mod_name, mod_fn
126 120
127 121 #-----------------------------------------------------------------------------
128 122 # Test automatic reloading
129 123 #-----------------------------------------------------------------------------
130 124
131 125 class TestAutoreload(Fixture):
132 126
133 127 @skipif(sys.version_info < (3, 6))
134 128 def test_reload_enums(self):
135 129 import enum
136 130 mod_name, mod_fn = self.new_module(textwrap.dedent("""
137 131 from enum import Enum
138 132 class MyEnum(Enum):
139 133 A = 'A'
140 134 B = 'B'
141 135 """))
142 136 self.shell.magic_autoreload("2")
143 137 self.shell.magic_aimport(mod_name)
144 138 self.write_file(mod_fn, textwrap.dedent("""
145 139 from enum import Enum
146 140 class MyEnum(Enum):
147 141 A = 'A'
148 142 B = 'B'
149 143 C = 'C'
150 144 """))
151 145 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
152 146 self.shell.run_code("pass") # trigger another reload
153 147
154 148 def test_reload_class_attributes(self):
155 149 self.shell.magic_autoreload("2")
156 150 mod_name, mod_fn = self.new_module(textwrap.dedent("""
157 151 class MyClass:
158 152
159 153 def __init__(self, a=10):
160 154 self.a = a
161 155 self.b = 22
162 156 # self.toto = 33
163 157
164 158 def square(self):
165 159 print('compute square')
166 160 return self.a*self.a
167 161 """
168 162 )
169 163 )
170 164 self.shell.run_code("from %s import MyClass" % mod_name)
171 165 self.shell.run_code("first = MyClass(5)")
172 166 self.shell.run_code("first.square()")
173 167 with nt.assert_raises(AttributeError):
174 168 self.shell.run_code("first.cube()")
175 169 with nt.assert_raises(AttributeError):
176 170 self.shell.run_code("first.power(5)")
177 171 self.shell.run_code("first.b")
178 172 with nt.assert_raises(AttributeError):
179 173 self.shell.run_code("first.toto")
180 174
181 175 # remove square, add power
182 176
183 177 self.write_file(
184 178 mod_fn,
185 179 textwrap.dedent(
186 180 """
187 181 class MyClass:
188 182
189 183 def __init__(self, a=10):
190 184 self.a = a
191 185 self.b = 11
192 186
193 187 def power(self, p):
194 188 print('compute power '+str(p))
195 189 return self.a**p
196 190 """
197 191 ),
198 192 )
199 193
200 194 self.shell.run_code("second = MyClass(5)")
201 195
202 196 for object_name in {'first', 'second'}:
203 197 self.shell.run_code("{object_name}.power(5)".format(object_name=object_name))
204 198 with nt.assert_raises(AttributeError):
205 199 self.shell.run_code("{object_name}.cube()".format(object_name=object_name))
206 200 with nt.assert_raises(AttributeError):
207 201 self.shell.run_code("{object_name}.square()".format(object_name=object_name))
208 202 self.shell.run_code("{object_name}.b".format(object_name=object_name))
209 203 self.shell.run_code("{object_name}.a".format(object_name=object_name))
210 204 with nt.assert_raises(AttributeError):
211 205 self.shell.run_code("{object_name}.toto".format(object_name=object_name))
212 206
213 207 def _check_smoketest(self, use_aimport=True):
214 208 """
215 209 Functional test for the automatic reloader using either
216 210 '%autoreload 1' or '%autoreload 2'
217 211 """
218 212
219 213 mod_name, mod_fn = self.new_module("""
220 214 x = 9
221 215
222 216 z = 123 # this item will be deleted
223 217
224 218 def foo(y):
225 219 return y + 3
226 220
227 221 class Baz(object):
228 222 def __init__(self, x):
229 223 self.x = x
230 224 def bar(self, y):
231 225 return self.x + y
232 226 @property
233 227 def quux(self):
234 228 return 42
235 229 def zzz(self):
236 230 '''This method will be deleted below'''
237 231 return 99
238 232
239 233 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
240 234 def foo(self):
241 235 return 1
242 236 """)
243 237
244 238 #
245 239 # Import module, and mark for reloading
246 240 #
247 241 if use_aimport:
248 242 self.shell.magic_autoreload("1")
249 243 self.shell.magic_aimport(mod_name)
250 244 stream = StringIO()
251 245 self.shell.magic_aimport("", stream=stream)
252 246 nt.assert_in(("Modules to reload:\n%s" % mod_name), stream.getvalue())
253 247
254 248 with nt.assert_raises(ImportError):
255 249 self.shell.magic_aimport("tmpmod_as318989e89ds")
256 250 else:
257 251 self.shell.magic_autoreload("2")
258 252 self.shell.run_code("import %s" % mod_name)
259 253 stream = StringIO()
260 254 self.shell.magic_aimport("", stream=stream)
261 255 nt.assert_true("Modules to reload:\nall-except-skipped" in
262 256 stream.getvalue())
263 257 nt.assert_in(mod_name, self.shell.ns)
264 258
265 259 mod = sys.modules[mod_name]
266 260
267 261 #
268 262 # Test module contents
269 263 #
270 264 old_foo = mod.foo
271 265 old_obj = mod.Baz(9)
272 266 old_obj2 = mod.Bar()
273 267
274 268 def check_module_contents():
275 269 nt.assert_equal(mod.x, 9)
276 270 nt.assert_equal(mod.z, 123)
277 271
278 272 nt.assert_equal(old_foo(0), 3)
279 273 nt.assert_equal(mod.foo(0), 3)
280 274
281 275 obj = mod.Baz(9)
282 276 nt.assert_equal(old_obj.bar(1), 10)
283 277 nt.assert_equal(obj.bar(1), 10)
284 278 nt.assert_equal(obj.quux, 42)
285 279 nt.assert_equal(obj.zzz(), 99)
286 280
287 281 obj2 = mod.Bar()
288 282 nt.assert_equal(old_obj2.foo(), 1)
289 283 nt.assert_equal(obj2.foo(), 1)
290 284
291 285 check_module_contents()
292 286
293 287 #
294 288 # Simulate a failed reload: no reload should occur and exactly
295 289 # one error message should be printed
296 290 #
297 291 self.write_file(mod_fn, """
298 292 a syntax error
299 293 """)
300 294
301 295 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
302 296 self.shell.run_code("pass") # trigger reload
303 297 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
304 298 self.shell.run_code("pass") # trigger another reload
305 299 check_module_contents()
306 300
307 301 #
308 302 # Rewrite module (this time reload should succeed)
309 303 #
310 304 self.write_file(mod_fn, """
311 305 x = 10
312 306
313 307 def foo(y):
314 308 return y + 4
315 309
316 310 class Baz(object):
317 311 def __init__(self, x):
318 312 self.x = x
319 313 def bar(self, y):
320 314 return self.x + y + 1
321 315 @property
322 316 def quux(self):
323 317 return 43
324 318
325 319 class Bar: # old-style class
326 320 def foo(self):
327 321 return 2
328 322 """)
329 323
330 324 def check_module_contents():
331 325 nt.assert_equal(mod.x, 10)
332 326 nt.assert_false(hasattr(mod, 'z'))
333 327
334 328 nt.assert_equal(old_foo(0), 4) # superreload magic!
335 329 nt.assert_equal(mod.foo(0), 4)
336 330
337 331 obj = mod.Baz(9)
338 332 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
339 333 nt.assert_equal(obj.bar(1), 11)
340 334
341 335 nt.assert_equal(old_obj.quux, 43)
342 336 nt.assert_equal(obj.quux, 43)
343 337
344 338 nt.assert_false(hasattr(old_obj, 'zzz'))
345 339 nt.assert_false(hasattr(obj, 'zzz'))
346 340
347 341 obj2 = mod.Bar()
348 342 nt.assert_equal(old_obj2.foo(), 2)
349 343 nt.assert_equal(obj2.foo(), 2)
350 344
351 345 self.shell.run_code("pass") # trigger reload
352 346 check_module_contents()
353 347
354 348 #
355 349 # Another failure case: deleted file (shouldn't reload)
356 350 #
357 351 os.unlink(mod_fn)
358 352
359 353 self.shell.run_code("pass") # trigger reload
360 354 check_module_contents()
361 355
362 356 #
363 357 # Disable autoreload and rewrite module: no reload should occur
364 358 #
365 359 if use_aimport:
366 360 self.shell.magic_aimport("-" + mod_name)
367 361 stream = StringIO()
368 362 self.shell.magic_aimport("", stream=stream)
369 363 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
370 364 stream.getvalue())
371 365
372 366 # This should succeed, although no such module exists
373 367 self.shell.magic_aimport("-tmpmod_as318989e89ds")
374 368 else:
375 369 self.shell.magic_autoreload("0")
376 370
377 371 self.write_file(mod_fn, """
378 372 x = -99
379 373 """)
380 374
381 375 self.shell.run_code("pass") # trigger reload
382 376 self.shell.run_code("pass")
383 377 check_module_contents()
384 378
385 379 #
386 380 # Re-enable autoreload: reload should now occur
387 381 #
388 382 if use_aimport:
389 383 self.shell.magic_aimport(mod_name)
390 384 else:
391 385 self.shell.magic_autoreload("")
392 386
393 387 self.shell.run_code("pass") # trigger reload
394 388 nt.assert_equal(mod.x, -99)
395 389
396 390 def test_smoketest_aimport(self):
397 391 self._check_smoketest(use_aimport=True)
398 392
399 393 def test_smoketest_autoreload(self):
400 394 self._check_smoketest(use_aimport=False)
401 395
402 396
403 397
404 398
@@ -1,764 +1,761 b''
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by setting the
7 7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 8 by interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import builtins as builtin_mod
23 23 import doctest
24 24 import inspect
25 25 import logging
26 26 import os
27 27 import re
28 28 import sys
29 29 from importlib import import_module
30 30 from io import StringIO
31 31
32 32 from testpath import modified_env
33 33
34 34 from inspect import getmodule
35 35
36 36 # We are overriding the default doctest runner, so we need to import a few
37 37 # things from doctest directly
38 38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 39 _unittest_reportflags, DocTestRunner,
40 40 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 41 _exception_traceback,
42 42 linecache)
43 43
44 44 # Third-party modules
45 45
46 46 from nose.plugins import doctests, Plugin
47 47 from nose.util import anyp, tolist
48 48
49 49 #-----------------------------------------------------------------------------
50 50 # Module globals and other constants
51 51 #-----------------------------------------------------------------------------
52 52
53 53 log = logging.getLogger(__name__)
54 54
55 55
56 56 #-----------------------------------------------------------------------------
57 57 # Classes and functions
58 58 #-----------------------------------------------------------------------------
59 59
60 60 def is_extension_module(filename):
61 61 """Return whether the given filename is an extension module.
62 62
63 63 This simply checks that the extension is either .so or .pyd.
64 64 """
65 65 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
66 66
67 67
68 68 class DocTestSkip(object):
69 69 """Object wrapper for doctests to be skipped."""
70 70
71 71 ds_skip = """Doctest to skip.
72 72 >>> 1 #doctest: +SKIP
73 73 """
74 74
75 75 def __init__(self,obj):
76 76 self.obj = obj
77 77
78 78 def __getattribute__(self,key):
79 79 if key == '__doc__':
80 80 return DocTestSkip.ds_skip
81 81 else:
82 82 return getattr(object.__getattribute__(self,'obj'),key)
83 83
84 84 # Modified version of the one in the stdlib, that fixes a python bug (doctests
85 85 # not found in extension modules, http://bugs.python.org/issue3158)
86 86 class DocTestFinder(doctest.DocTestFinder):
87 87
88 88 def _from_module(self, module, object):
89 89 """
90 90 Return true if the given object is defined in the given
91 91 module.
92 92 """
93 93 if module is None:
94 94 return True
95 95 elif inspect.isfunction(object):
96 96 return module.__dict__ is object.__globals__
97 97 elif inspect.isbuiltin(object):
98 98 return module.__name__ == object.__module__
99 99 elif inspect.isclass(object):
100 100 return module.__name__ == object.__module__
101 101 elif inspect.ismethod(object):
102 102 # This one may be a bug in cython that fails to correctly set the
103 103 # __module__ attribute of methods, but since the same error is easy
104 104 # to make by extension code writers, having this safety in place
105 105 # isn't such a bad idea
106 106 return module.__name__ == object.__self__.__class__.__module__
107 107 elif inspect.getmodule(object) is not None:
108 108 return module is inspect.getmodule(object)
109 109 elif hasattr(object, '__module__'):
110 110 return module.__name__ == object.__module__
111 111 elif isinstance(object, property):
112 112 return True # [XX] no way not be sure.
113 113 elif inspect.ismethoddescriptor(object):
114 114 # Unbound PyQt signals reach this point in Python 3.4b3, and we want
115 115 # to avoid throwing an error. See also http://bugs.python.org/issue3158
116 116 return False
117 117 else:
118 118 raise ValueError("object must be a class or function, got %r" % object)
119 119
120 120 def _find(self, tests, obj, name, module, source_lines, globs, seen):
121 121 """
122 122 Find tests for the given object and any contained objects, and
123 123 add them to `tests`.
124 124 """
125 125 print('_find for:', obj, name, module) # dbg
126 126 if hasattr(obj,"skip_doctest"):
127 127 #print 'SKIPPING DOCTEST FOR:',obj # dbg
128 128 obj = DocTestSkip(obj)
129 129
130 130 doctest.DocTestFinder._find(self,tests, obj, name, module,
131 131 source_lines, globs, seen)
132 132
133 133 # Below we re-run pieces of the above method with manual modifications,
134 134 # because the original code is buggy and fails to correctly identify
135 135 # doctests in extension modules.
136 136
137 137 # Local shorthands
138 138 from inspect import isroutine, isclass
139 139
140 140 # Look for tests in a module's contained objects.
141 141 if inspect.ismodule(obj) and self._recurse:
142 142 for valname, val in obj.__dict__.items():
143 143 valname1 = '%s.%s' % (name, valname)
144 144 if ( (isroutine(val) or isclass(val))
145 145 and self._from_module(module, val) ):
146 146
147 147 self._find(tests, val, valname1, module, source_lines,
148 148 globs, seen)
149 149
150 150 # Look for tests in a class's contained objects.
151 151 if inspect.isclass(obj) and self._recurse:
152 152 #print 'RECURSE into class:',obj # dbg
153 153 for valname, val in obj.__dict__.items():
154 154 # Special handling for staticmethod/classmethod.
155 155 if isinstance(val, staticmethod):
156 156 val = getattr(obj, valname)
157 157 if isinstance(val, classmethod):
158 158 val = getattr(obj, valname).__func__
159 159
160 160 # Recurse to methods, properties, and nested classes.
161 161 if ((inspect.isfunction(val) or inspect.isclass(val) or
162 162 inspect.ismethod(val) or
163 163 isinstance(val, property)) and
164 164 self._from_module(module, val)):
165 165 valname = '%s.%s' % (name, valname)
166 166 self._find(tests, val, valname, module, source_lines,
167 167 globs, seen)
168 168
169 169
170 170 class IPDoctestOutputChecker(doctest.OutputChecker):
171 171 """Second-chance checker with support for random tests.
172 172
173 173 If the default comparison doesn't pass, this checker looks in the expected
174 174 output string for flags that tell us to ignore the output.
175 175 """
176 176
177 177 random_re = re.compile(r'#\s*random\s+')
178 178
179 179 def check_output(self, want, got, optionflags):
180 180 """Check output, accepting special markers embedded in the output.
181 181
182 182 If the output didn't pass the default validation but the special string
183 183 '#random' is included, we accept it."""
184 184
185 185 # Let the original tester verify first, in case people have valid tests
186 186 # that happen to have a comment saying '#random' embedded in.
187 187 ret = doctest.OutputChecker.check_output(self, want, got,
188 188 optionflags)
189 189 if not ret and self.random_re.search(want):
190 190 #print >> sys.stderr, 'RANDOM OK:',want # dbg
191 191 return True
192 192
193 193 return ret
194 194
195 195
196 196 class DocTestCase(doctests.DocTestCase):
197 197 """Proxy for DocTestCase: provides an address() method that
198 198 returns the correct address for the doctest case. Otherwise
199 199 acts as a proxy to the test case. To provide hints for address(),
200 200 an obj may also be passed -- this will be used as the test object
201 201 for purposes of determining the test address, if it is provided.
202 202 """
203 203
204 204 # Note: this method was taken from numpy's nosetester module.
205 205
206 206 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
207 207 # its constructor that blocks non-default arguments from being passed
208 208 # down into doctest.DocTestCase
209 209
210 210 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
211 211 checker=None, obj=None, result_var='_'):
212 212 self._result_var = result_var
213 213 doctests.DocTestCase.__init__(self, test,
214 214 optionflags=optionflags,
215 215 setUp=setUp, tearDown=tearDown,
216 216 checker=checker)
217 217 # Now we must actually copy the original constructor from the stdlib
218 218 # doctest class, because we can't call it directly and a bug in nose
219 219 # means it never gets passed the right arguments.
220 220
221 221 self._dt_optionflags = optionflags
222 222 self._dt_checker = checker
223 223 self._dt_test = test
224 224 self._dt_test_globs_ori = test.globs
225 225 self._dt_setUp = setUp
226 226 self._dt_tearDown = tearDown
227 227
228 228 # XXX - store this runner once in the object!
229 229 runner = IPDocTestRunner(optionflags=optionflags,
230 230 checker=checker, verbose=False)
231 231 self._dt_runner = runner
232 232
233 233
234 234 # Each doctest should remember the directory it was loaded from, so
235 235 # things like %run work without too many contortions
236 236 self._ori_dir = os.path.dirname(test.filename)
237 237
238 238 # Modified runTest from the default stdlib
239 239 def runTest(self):
240 240 test = self._dt_test
241 241 runner = self._dt_runner
242 242
243 243 old = sys.stdout
244 244 new = StringIO()
245 245 optionflags = self._dt_optionflags
246 246
247 247 if not (optionflags & REPORTING_FLAGS):
248 248 # The option flags don't include any reporting flags,
249 249 # so add the default reporting flags
250 250 optionflags |= _unittest_reportflags
251 251
252 252 try:
253 253 # Save our current directory and switch out to the one where the
254 254 # test was originally created, in case another doctest did a
255 255 # directory change. We'll restore this in the finally clause.
256 256 curdir = os.getcwd()
257 257 #print 'runTest in dir:', self._ori_dir # dbg
258 258 os.chdir(self._ori_dir)
259 259
260 260 runner.DIVIDER = "-"*70
261 261 failures, tries = runner.run(test,out=new.write,
262 262 clear_globs=False)
263 263 finally:
264 264 sys.stdout = old
265 265 os.chdir(curdir)
266 266
267 267 if failures:
268 268 raise self.failureException(self.format_failure(new.getvalue()))
269 269
270 270 def setUp(self):
271 271 """Modified test setup that syncs with ipython namespace"""
272 272 #print "setUp test", self._dt_test.examples # dbg
273 273 if isinstance(self._dt_test.examples[0], IPExample):
274 274 # for IPython examples *only*, we swap the globals with the ipython
275 275 # namespace, after updating it with the globals (which doctest
276 276 # fills with the necessary info from the module being tested).
277 277 self.user_ns_orig = {}
278 278 self.user_ns_orig.update(_ip.user_ns)
279 279 _ip.user_ns.update(self._dt_test.globs)
280 280 # We must remove the _ key in the namespace, so that Python's
281 281 # doctest code sets it naturally
282 282 _ip.user_ns.pop('_', None)
283 283 _ip.user_ns['__builtins__'] = builtin_mod
284 284 self._dt_test.globs = _ip.user_ns
285 285
286 286 super(DocTestCase, self).setUp()
287 287
288 288 def tearDown(self):
289 289
290 290 # Undo the test.globs reassignment we made, so that the parent class
291 291 # teardown doesn't destroy the ipython namespace
292 292 if isinstance(self._dt_test.examples[0], IPExample):
293 293 self._dt_test.globs = self._dt_test_globs_ori
294 294 _ip.user_ns.clear()
295 295 _ip.user_ns.update(self.user_ns_orig)
296 296
297 297 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
298 298 # it does look like one to me: its tearDown method tries to run
299 299 #
300 300 # delattr(builtin_mod, self._result_var)
301 301 #
302 302 # without checking that the attribute really is there; it implicitly
303 303 # assumes it should have been set via displayhook. But if the
304 304 # displayhook was never called, this doesn't necessarily happen. I
305 305 # haven't been able to find a little self-contained example outside of
306 306 # ipython that would show the problem so I can report it to the nose
307 307 # team, but it does happen a lot in our code.
308 308 #
309 309 # So here, we just protect as narrowly as possible by trapping an
310 310 # attribute error whose message would be the name of self._result_var,
311 311 # and letting any other error propagate.
312 312 try:
313 313 super(DocTestCase, self).tearDown()
314 314 except AttributeError as exc:
315 315 if exc.args[0] != self._result_var:
316 316 raise
317 317
318 318
319 319 # A simple subclassing of the original with a different class name, so we can
320 320 # distinguish and treat differently IPython examples from pure python ones.
321 321 class IPExample(doctest.Example): pass
322 322
323 323
324 324 class IPExternalExample(doctest.Example):
325 325 """Doctest examples to be run in an external process."""
326 326
327 327 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
328 328 options=None):
329 329 # Parent constructor
330 330 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
331 331
332 332 # An EXTRA newline is needed to prevent pexpect hangs
333 333 self.source += '\n'
334 334
335 335
336 336 class IPDocTestParser(doctest.DocTestParser):
337 337 """
338 338 A class used to parse strings containing doctest examples.
339 339
340 340 Note: This is a version modified to properly recognize IPython input and
341 341 convert any IPython examples into valid Python ones.
342 342 """
343 343 # This regular expression is used to find doctest examples in a
344 344 # string. It defines three groups: `source` is the source code
345 345 # (including leading indentation and prompts); `indent` is the
346 346 # indentation of the first (PS1) line of the source code; and
347 347 # `want` is the expected output (including leading indentation).
348 348
349 349 # Classic Python prompts or default IPython ones
350 350 _PS1_PY = r'>>>'
351 351 _PS2_PY = r'\.\.\.'
352 352
353 353 _PS1_IP = r'In\ \[\d+\]:'
354 354 _PS2_IP = r'\ \ \ \.\.\.+:'
355 355
356 356 _RE_TPL = r'''
357 357 # Source consists of a PS1 line followed by zero or more PS2 lines.
358 358 (?P<source>
359 359 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
360 360 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
361 361 \n? # a newline
362 362 # Want consists of any non-blank lines that do not start with PS1.
363 363 (?P<want> (?:(?![ ]*$) # Not a blank line
364 364 (?![ ]*%s) # Not a line starting with PS1
365 365 (?![ ]*%s) # Not a line starting with PS2
366 366 .*$\n? # But any other line
367 367 )*)
368 368 '''
369 369
370 370 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
371 371 re.MULTILINE | re.VERBOSE)
372 372
373 373 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
374 374 re.MULTILINE | re.VERBOSE)
375 375
376 376 # Mark a test as being fully random. In this case, we simply append the
377 377 # random marker ('#random') to each individual example's output. This way
378 378 # we don't need to modify any other code.
379 379 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
380 380
381 381 # Mark tests to be executed in an external process - currently unsupported.
382 382 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
383 383
384 384 def ip2py(self,source):
385 385 """Convert input IPython source into valid Python."""
386 386 block = _ip.input_transformer_manager.transform_cell(source)
387 387 if len(block.splitlines()) == 1:
388 388 return _ip.prefilter(block)
389 389 else:
390 390 return block
391 391
392 392 def parse(self, string, name='<string>'):
393 393 """
394 394 Divide the given string into examples and intervening text,
395 395 and return them as a list of alternating Examples and strings.
396 396 Line numbers for the Examples are 0-based. The optional
397 397 argument `name` is a name identifying this string, and is only
398 398 used for error messages.
399 399 """
400 400
401 401 #print 'Parse string:\n',string # dbg
402 402
403 403 string = string.expandtabs()
404 404 # If all lines begin with the same indentation, then strip it.
405 405 min_indent = self._min_indent(string)
406 406 if min_indent > 0:
407 407 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
408 408
409 409 output = []
410 410 charno, lineno = 0, 0
411 411
412 412 # We make 'all random' tests by adding the '# random' mark to every
413 413 # block of output in the test.
414 414 if self._RANDOM_TEST.search(string):
415 415 random_marker = '\n# random'
416 416 else:
417 417 random_marker = ''
418 418
419 419 # Whether to convert the input from ipython to python syntax
420 420 ip2py = False
421 421 # Find all doctest examples in the string. First, try them as Python
422 422 # examples, then as IPython ones
423 423 terms = list(self._EXAMPLE_RE_PY.finditer(string))
424 424 if terms:
425 425 # Normal Python example
426 426 #print '-'*70 # dbg
427 427 #print 'PyExample, Source:\n',string # dbg
428 428 #print '-'*70 # dbg
429 429 Example = doctest.Example
430 430 else:
431 431 # It's an ipython example. Note that IPExamples are run
432 432 # in-process, so their syntax must be turned into valid python.
433 433 # IPExternalExamples are run out-of-process (via pexpect) so they
434 434 # don't need any filtering (a real ipython will be executing them).
435 435 terms = list(self._EXAMPLE_RE_IP.finditer(string))
436 436 if self._EXTERNAL_IP.search(string):
437 437 #print '-'*70 # dbg
438 438 #print 'IPExternalExample, Source:\n',string # dbg
439 439 #print '-'*70 # dbg
440 440 Example = IPExternalExample
441 441 else:
442 442 #print '-'*70 # dbg
443 443 #print 'IPExample, Source:\n',string # dbg
444 444 #print '-'*70 # dbg
445 445 Example = IPExample
446 446 ip2py = True
447 447
448 448 for m in terms:
449 449 # Add the pre-example text to `output`.
450 450 output.append(string[charno:m.start()])
451 451 # Update lineno (lines before this example)
452 452 lineno += string.count('\n', charno, m.start())
453 453 # Extract info from the regexp match.
454 454 (source, options, want, exc_msg) = \
455 455 self._parse_example(m, name, lineno,ip2py)
456 456
457 457 # Append the random-output marker (it defaults to empty in most
458 458 # cases, it's only non-empty for 'all-random' tests):
459 459 want += random_marker
460 460
461 461 if Example is IPExternalExample:
462 462 options[doctest.NORMALIZE_WHITESPACE] = True
463 463 want += '\n'
464 464
465 465 # Create an Example, and add it to the list.
466 466 if not self._IS_BLANK_OR_COMMENT(source):
467 467 output.append(Example(source, want, exc_msg,
468 468 lineno=lineno,
469 469 indent=min_indent+len(m.group('indent')),
470 470 options=options))
471 471 # Update lineno (lines inside this example)
472 472 lineno += string.count('\n', m.start(), m.end())
473 473 # Update charno.
474 474 charno = m.end()
475 475 # Add any remaining post-example text to `output`.
476 476 output.append(string[charno:])
477 477 return output
478 478
479 479 def _parse_example(self, m, name, lineno,ip2py=False):
480 480 """
481 481 Given a regular expression match from `_EXAMPLE_RE` (`m`),
482 482 return a pair `(source, want)`, where `source` is the matched
483 483 example's source code (with prompts and indentation stripped);
484 484 and `want` is the example's expected output (with indentation
485 485 stripped).
486 486
487 487 `name` is the string's name, and `lineno` is the line number
488 488 where the example starts; both are used for error messages.
489 489
490 490 Optional:
491 491 `ip2py`: if true, filter the input via IPython to convert the syntax
492 492 into valid python.
493 493 """
494 494
495 495 # Get the example's indentation level.
496 496 indent = len(m.group('indent'))
497 497
498 498 # Divide source into lines; check that they're properly
499 499 # indented; and then strip their indentation & prompts.
500 500 source_lines = m.group('source').split('\n')
501 501
502 502 # We're using variable-length input prompts
503 503 ps1 = m.group('ps1')
504 504 ps2 = m.group('ps2')
505 505 ps1_len = len(ps1)
506 506
507 507 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
508 508 if ps2:
509 509 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
510 510
511 511 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
512 512
513 513 if ip2py:
514 514 # Convert source input from IPython into valid Python syntax
515 515 source = self.ip2py(source)
516 516
517 517 # Divide want into lines; check that it's properly indented; and
518 518 # then strip the indentation. Spaces before the last newline should
519 519 # be preserved, so plain rstrip() isn't good enough.
520 520 want = m.group('want')
521 521 want_lines = want.split('\n')
522 522 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
523 523 del want_lines[-1] # forget final newline & spaces after it
524 524 self._check_prefix(want_lines, ' '*indent, name,
525 525 lineno + len(source_lines))
526 526
527 527 # Remove ipython output prompt that might be present in the first line
528 528 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
529 529
530 530 want = '\n'.join([wl[indent:] for wl in want_lines])
531 531
532 532 # If `want` contains a traceback message, then extract it.
533 533 m = self._EXCEPTION_RE.match(want)
534 534 if m:
535 535 exc_msg = m.group('msg')
536 536 else:
537 537 exc_msg = None
538 538
539 539 # Extract options from the source.
540 540 options = self._find_options(source, name, lineno)
541 541
542 542 return source, options, want, exc_msg
543 543
544 544 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
545 545 """
546 546 Given the lines of a source string (including prompts and
547 547 leading indentation), check to make sure that every prompt is
548 548 followed by a space character. If any line is not followed by
549 549 a space character, then raise ValueError.
550 550
551 551 Note: IPython-modified version which takes the input prompt length as a
552 552 parameter, so that prompts of variable length can be dealt with.
553 553 """
554 554 space_idx = indent+ps1_len
555 555 min_len = space_idx+1
556 556 for i, line in enumerate(lines):
557 557 if len(line) >= min_len and line[space_idx] != ' ':
558 558 raise ValueError('line %r of the docstring for %s '
559 559 'lacks blank after %s: %r' %
560 560 (lineno+i+1, name,
561 561 line[indent:space_idx], line))
562 562
563 563
564 564 SKIP = doctest.register_optionflag('SKIP')
565 565
566 566
567 567 class IPDocTestRunner(doctest.DocTestRunner,object):
568 568 """Test runner that synchronizes the IPython namespace with test globals.
569 569 """
570 570
571 571 def run(self, test, compileflags=None, out=None, clear_globs=True):
572 572
573 573 # Hack: ipython needs access to the execution context of the example,
574 574 # so that it can propagate user variables loaded by %run into
575 575 # test.globs. We put them here into our modified %run as a function
576 576 # attribute. Our new %run will then only make the namespace update
577 577 # when called (rather than unconconditionally updating test.globs here
578 578 # for all examples, most of which won't be calling %run anyway).
579 579 #_ip._ipdoctest_test_globs = test.globs
580 580 #_ip._ipdoctest_test_filename = test.filename
581 581
582 582 test.globs.update(_ip.user_ns)
583 583
584 584 # Override terminal size to standardise traceback format
585 585 with modified_env({'COLUMNS': '80', 'LINES': '24'}):
586 586 return super(IPDocTestRunner,self).run(test,
587 587 compileflags,out,clear_globs)
588 588
589 589
590 590 class DocFileCase(doctest.DocFileCase):
591 591 """Overrides to provide filename
592 592 """
593 593 def address(self):
594 594 return (self._dt_test.filename, None, None)
595 595
596 596
597 597 class ExtensionDoctest(doctests.Doctest):
598 598 """Nose Plugin that supports doctests in extension modules.
599 599 """
600 600 name = 'extdoctest' # call nosetests with --with-extdoctest
601 601 enabled = True
602 602
603 603 def options(self, parser, env=os.environ):
604 604 Plugin.options(self, parser, env)
605 605 parser.add_option('--doctest-tests', action='store_true',
606 606 dest='doctest_tests',
607 607 default=env.get('NOSE_DOCTEST_TESTS',True),
608 608 help="Also look for doctests in test modules. "
609 609 "Note that classes, methods and functions should "
610 610 "have either doctests or non-doctest tests, "
611 611 "not both. [NOSE_DOCTEST_TESTS]")
612 612 parser.add_option('--doctest-extension', action="append",
613 613 dest="doctestExtension",
614 614 help="Also look for doctests in files with "
615 615 "this extension [NOSE_DOCTEST_EXTENSION]")
616 616 # Set the default as a list, if given in env; otherwise
617 617 # an additional value set on the command line will cause
618 618 # an error.
619 619 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
620 620 if env_setting is not None:
621 621 parser.set_defaults(doctestExtension=tolist(env_setting))
622 622
623 623
624 624 def configure(self, options, config):
625 625 Plugin.configure(self, options, config)
626 626 # Pull standard doctest plugin out of config; we will do doctesting
627 627 config.plugins.plugins = [p for p in config.plugins.plugins
628 628 if p.name != 'doctest']
629 629 self.doctest_tests = options.doctest_tests
630 630 self.extension = tolist(options.doctestExtension)
631 631
632 632 self.parser = doctest.DocTestParser()
633 633 self.finder = DocTestFinder()
634 634 self.checker = IPDoctestOutputChecker()
635 635 self.globs = None
636 636 self.extraglobs = None
637 637
638 638
639 639 def loadTestsFromExtensionModule(self,filename):
640 640 bpath,mod = os.path.split(filename)
641 641 modname = os.path.splitext(mod)[0]
642 642 try:
643 643 sys.path.append(bpath)
644 644 module = import_module(modname)
645 645 tests = list(self.loadTestsFromModule(module))
646 646 finally:
647 647 sys.path.pop()
648 648 return tests
649 649
650 650 # NOTE: the method below is almost a copy of the original one in nose, with
651 651 # a few modifications to control output checking.
652 652
653 653 def loadTestsFromModule(self, module):
654 654 #print '*** ipdoctest - lTM',module # dbg
655 655
656 656 if not self.matches(module.__name__):
657 657 log.debug("Doctest doesn't want module %s", module)
658 658 return
659 659
660 660 tests = self.finder.find(module,globs=self.globs,
661 661 extraglobs=self.extraglobs)
662 662 if not tests:
663 663 return
664 664
665 665 # always use whitespace and ellipsis options
666 666 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
667 667
668 668 tests.sort()
669 669 module_file = module.__file__
670 670 if module_file[-4:] in ('.pyc', '.pyo'):
671 671 module_file = module_file[:-1]
672 672 for test in tests:
673 673 if not test.examples:
674 674 continue
675 675 if not test.filename:
676 676 test.filename = module_file
677 677
678 678 yield DocTestCase(test,
679 679 optionflags=optionflags,
680 680 checker=self.checker)
681 681
682 682
683 683 def loadTestsFromFile(self, filename):
684 684 #print "ipdoctest - from file", filename # dbg
685 685 if is_extension_module(filename):
686 686 for t in self.loadTestsFromExtensionModule(filename):
687 687 yield t
688 688 else:
689 689 if self.extension and anyp(filename.endswith, self.extension):
690 690 name = os.path.basename(filename)
691 dh = open(filename)
692 try:
691 with open(filename) as dh:
693 692 doc = dh.read()
694 finally:
695 dh.close()
696 693 test = self.parser.get_doctest(
697 694 doc, globs={'__file__': filename}, name=name,
698 695 filename=filename, lineno=0)
699 696 if test.examples:
700 697 #print 'FileCase:',test.examples # dbg
701 698 yield DocFileCase(test)
702 699 else:
703 700 yield False # no tests to load
704 701
705 702
706 703 class IPythonDoctest(ExtensionDoctest):
707 704 """Nose Plugin that supports doctests in extension modules.
708 705 """
709 706 name = 'ipdoctest' # call nosetests with --with-ipdoctest
710 707 enabled = True
711 708
712 709 def makeTest(self, obj, parent):
713 710 """Look for doctests in the given object, which will be a
714 711 function, method or class.
715 712 """
716 713 #print 'Plugin analyzing:', obj, parent # dbg
717 714 # always use whitespace and ellipsis options
718 715 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
719 716
720 717 doctests = self.finder.find(obj, module=getmodule(parent))
721 718 if doctests:
722 719 for test in doctests:
723 720 if len(test.examples) == 0:
724 721 continue
725 722
726 723 yield DocTestCase(test, obj=obj,
727 724 optionflags=optionflags,
728 725 checker=self.checker)
729 726
730 727 def options(self, parser, env=os.environ):
731 728 #print "Options for nose plugin:", self.name # dbg
732 729 Plugin.options(self, parser, env)
733 730 parser.add_option('--ipdoctest-tests', action='store_true',
734 731 dest='ipdoctest_tests',
735 732 default=env.get('NOSE_IPDOCTEST_TESTS',True),
736 733 help="Also look for doctests in test modules. "
737 734 "Note that classes, methods and functions should "
738 735 "have either doctests or non-doctest tests, "
739 736 "not both. [NOSE_IPDOCTEST_TESTS]")
740 737 parser.add_option('--ipdoctest-extension', action="append",
741 738 dest="ipdoctest_extension",
742 739 help="Also look for doctests in files with "
743 740 "this extension [NOSE_IPDOCTEST_EXTENSION]")
744 741 # Set the default as a list, if given in env; otherwise
745 742 # an additional value set on the command line will cause
746 743 # an error.
747 744 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
748 745 if env_setting is not None:
749 746 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
750 747
751 748 def configure(self, options, config):
752 749 #print "Configuring nose plugin:", self.name # dbg
753 750 Plugin.configure(self, options, config)
754 751 # Pull standard doctest plugin out of config; we will do doctesting
755 752 config.plugins.plugins = [p for p in config.plugins.plugins
756 753 if p.name != 'doctest']
757 754 self.doctest_tests = options.ipdoctest_tests
758 755 self.extension = tolist(options.ipdoctest_extension)
759 756
760 757 self.parser = IPDocTestParser()
761 758 self.finder = DocTestFinder(parser=self.parser)
762 759 self.checker = IPDoctestOutputChecker()
763 760 self.globs = None
764 761 self.extraglobs = None
@@ -1,472 +1,471 b''
1 1 """Generic testing tools.
2 2
3 3 Authors
4 4 -------
5 5 - Fernando Perez <Fernando.Perez@berkeley.edu>
6 6 """
7 7
8 8
9 9 # Copyright (c) IPython Development Team.
10 10 # Distributed under the terms of the Modified BSD License.
11 11
12 12 import os
13 13 import re
14 14 import sys
15 15 import tempfile
16 16
17 17 from contextlib import contextmanager
18 18 from io import StringIO
19 19 from subprocess import Popen, PIPE
20 20 from unittest.mock import patch
21 21
22 22 try:
23 23 # These tools are used by parts of the runtime, so we make the nose
24 24 # dependency optional at this point. Nose is a hard dependency to run the
25 25 # test suite, but NOT to use ipython itself.
26 26 import nose.tools as nt
27 27 has_nose = True
28 28 except ImportError:
29 29 has_nose = False
30 30
31 31 from traitlets.config.loader import Config
32 32 from IPython.utils.process import get_output_error_code
33 33 from IPython.utils.text import list_strings
34 34 from IPython.utils.io import temp_pyfile, Tee
35 35 from IPython.utils import py3compat
36 36
37 37 from . import decorators as dec
38 38 from . import skipdoctest
39 39
40 40
41 41 # The docstring for full_path doctests differently on win32 (different path
42 42 # separator) so just skip the doctest there. The example remains informative.
43 43 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
44 44
45 45 @doctest_deco
46 46 def full_path(startPath,files):
47 47 """Make full paths for all the listed files, based on startPath.
48 48
49 49 Only the base part of startPath is kept, since this routine is typically
50 50 used with a script's ``__file__`` variable as startPath. The base of startPath
51 51 is then prepended to all the listed files, forming the output list.
52 52
53 53 Parameters
54 54 ----------
55 55 startPath : string
56 56 Initial path to use as the base for the results. This path is split
57 57 using os.path.split() and only its first component is kept.
58 58
59 59 files : string or list
60 60 One or more files.
61 61
62 62 Examples
63 63 --------
64 64
65 65 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
66 66 ['/foo/a.txt', '/foo/b.txt']
67 67
68 68 >>> full_path('/foo',['a.txt','b.txt'])
69 69 ['/a.txt', '/b.txt']
70 70
71 71 If a single file is given, the output is still a list::
72 72
73 73 >>> full_path('/foo','a.txt')
74 74 ['/a.txt']
75 75 """
76 76
77 77 files = list_strings(files)
78 78 base = os.path.split(startPath)[0]
79 79 return [ os.path.join(base,f) for f in files ]
80 80
81 81
82 82 def parse_test_output(txt):
83 83 """Parse the output of a test run and return errors, failures.
84 84
85 85 Parameters
86 86 ----------
87 87 txt : str
88 88 Text output of a test run, assumed to contain a line of one of the
89 89 following forms::
90 90
91 91 'FAILED (errors=1)'
92 92 'FAILED (failures=1)'
93 93 'FAILED (errors=1, failures=1)'
94 94
95 95 Returns
96 96 -------
97 97 nerr, nfail
98 98 number of errors and failures.
99 99 """
100 100
101 101 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
102 102 if err_m:
103 103 nerr = int(err_m.group(1))
104 104 nfail = 0
105 105 return nerr, nfail
106 106
107 107 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
108 108 if fail_m:
109 109 nerr = 0
110 110 nfail = int(fail_m.group(1))
111 111 return nerr, nfail
112 112
113 113 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
114 114 re.MULTILINE)
115 115 if both_m:
116 116 nerr = int(both_m.group(1))
117 117 nfail = int(both_m.group(2))
118 118 return nerr, nfail
119 119
120 120 # If the input didn't match any of these forms, assume no error/failures
121 121 return 0, 0
122 122
123 123
124 124 # So nose doesn't think this is a test
125 125 parse_test_output.__test__ = False
126 126
127 127
128 128 def default_argv():
129 129 """Return a valid default argv for creating testing instances of ipython"""
130 130
131 131 return ['--quick', # so no config file is loaded
132 132 # Other defaults to minimize side effects on stdout
133 133 '--colors=NoColor', '--no-term-title','--no-banner',
134 134 '--autocall=0']
135 135
136 136
137 137 def default_config():
138 138 """Return a config object with good defaults for testing."""
139 139 config = Config()
140 140 config.TerminalInteractiveShell.colors = 'NoColor'
141 141 config.TerminalTerminalInteractiveShell.term_title = False,
142 142 config.TerminalInteractiveShell.autocall = 0
143 143 f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
144 144 config.HistoryManager.hist_file = f.name
145 145 f.close()
146 146 config.HistoryManager.db_cache_size = 10000
147 147 return config
148 148
149 149
150 150 def get_ipython_cmd(as_string=False):
151 151 """
152 152 Return appropriate IPython command line name. By default, this will return
153 153 a list that can be used with subprocess.Popen, for example, but passing
154 154 `as_string=True` allows for returning the IPython command as a string.
155 155
156 156 Parameters
157 157 ----------
158 158 as_string: bool
159 159 Flag to allow to return the command as a string.
160 160 """
161 161 ipython_cmd = [sys.executable, "-m", "IPython"]
162 162
163 163 if as_string:
164 164 ipython_cmd = " ".join(ipython_cmd)
165 165
166 166 return ipython_cmd
167 167
168 168 def ipexec(fname, options=None, commands=()):
169 169 """Utility to call 'ipython filename'.
170 170
171 171 Starts IPython with a minimal and safe configuration to make startup as fast
172 172 as possible.
173 173
174 174 Note that this starts IPython in a subprocess!
175 175
176 176 Parameters
177 177 ----------
178 178 fname : str
179 179 Name of file to be executed (should have .py or .ipy extension).
180 180
181 181 options : optional, list
182 182 Extra command-line flags to be passed to IPython.
183 183
184 184 commands : optional, list
185 185 Commands to send in on stdin
186 186
187 187 Returns
188 188 -------
189 189 ``(stdout, stderr)`` of ipython subprocess.
190 190 """
191 191 if options is None: options = []
192 192
193 193 cmdargs = default_argv() + options
194 194
195 195 test_dir = os.path.dirname(__file__)
196 196
197 197 ipython_cmd = get_ipython_cmd()
198 198 # Absolute path for filename
199 199 full_fname = os.path.join(test_dir, fname)
200 200 full_cmd = ipython_cmd + cmdargs + [full_fname]
201 201 env = os.environ.copy()
202 202 # FIXME: ignore all warnings in ipexec while we have shims
203 203 # should we keep suppressing warnings here, even after removing shims?
204 204 env['PYTHONWARNINGS'] = 'ignore'
205 205 # env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
206 206 for k, v in env.items():
207 207 # Debug a bizarre failure we've seen on Windows:
208 208 # TypeError: environment can only contain strings
209 209 if not isinstance(v, str):
210 210 print(k, v)
211 211 p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
212 212 out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None)
213 213 out, err = py3compat.decode(out), py3compat.decode(err)
214 214 # `import readline` causes 'ESC[?1034h' to be output sometimes,
215 215 # so strip that out before doing comparisons
216 216 if out:
217 217 out = re.sub(r'\x1b\[[^h]+h', '', out)
218 218 return out, err
219 219
220 220
221 221 def ipexec_validate(fname, expected_out, expected_err='',
222 222 options=None, commands=()):
223 223 """Utility to call 'ipython filename' and validate output/error.
224 224
225 225 This function raises an AssertionError if the validation fails.
226 226
227 227 Note that this starts IPython in a subprocess!
228 228
229 229 Parameters
230 230 ----------
231 231 fname : str
232 232 Name of the file to be executed (should have .py or .ipy extension).
233 233
234 234 expected_out : str
235 235 Expected stdout of the process.
236 236
237 237 expected_err : optional, str
238 238 Expected stderr of the process.
239 239
240 240 options : optional, list
241 241 Extra command-line flags to be passed to IPython.
242 242
243 243 Returns
244 244 -------
245 245 None
246 246 """
247 247
248 248 import nose.tools as nt
249 249
250 250 out, err = ipexec(fname, options, commands)
251 251 #print 'OUT', out # dbg
252 252 #print 'ERR', err # dbg
253 253 # If there are any errors, we must check those before stdout, as they may be
254 254 # more informative than simply having an empty stdout.
255 255 if err:
256 256 if expected_err:
257 257 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
258 258 else:
259 259 raise ValueError('Running file %r produced error: %r' %
260 260 (fname, err))
261 261 # If no errors or output on stderr was expected, match stdout
262 262 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
263 263
264 264
265 265 class TempFileMixin(object):
266 266 """Utility class to create temporary Python/IPython files.
267 267
268 268 Meant as a mixin class for test cases."""
269 269
270 270 def mktmp(self, src, ext='.py'):
271 271 """Make a valid python temp file."""
272 272 fname, f = temp_pyfile(src, ext)
273 273 if not hasattr(self, 'tmps'):
274 274 self.tmps=[]
275 275 self.tmps.append((f, fname))
276 276 self.fname = fname
277 277
278 278 def tearDown(self):
279 279 # If the tmpfile wasn't made because of skipped tests, like in
280 280 # win32, there's nothing to cleanup.
281 281 if hasattr(self, 'tmps'):
282 282 for f,fname in self.tmps:
283 283 # If the tmpfile wasn't made because of skipped tests, like in
284 284 # win32, there's nothing to cleanup.
285 285 f.close()
286 286 try:
287 287 os.unlink(fname)
288 288 except:
289 289 # On Windows, even though we close the file, we still can't
290 290 # delete it. I have no clue why
291 291 pass
292 292
293 293 def __enter__(self):
294 294 return self
295 295
296 296 def __exit__(self, exc_type, exc_value, traceback):
297 297 self.tearDown()
298 298
299 299
300 300 pair_fail_msg = ("Testing {0}\n\n"
301 301 "In:\n"
302 302 " {1!r}\n"
303 303 "Expected:\n"
304 304 " {2!r}\n"
305 305 "Got:\n"
306 306 " {3!r}\n")
307 307 def check_pairs(func, pairs):
308 308 """Utility function for the common case of checking a function with a
309 309 sequence of input/output pairs.
310 310
311 311 Parameters
312 312 ----------
313 313 func : callable
314 314 The function to be tested. Should accept a single argument.
315 315 pairs : iterable
316 316 A list of (input, expected_output) tuples.
317 317
318 318 Returns
319 319 -------
320 320 None. Raises an AssertionError if any output does not match the expected
321 321 value.
322 322 """
323 323 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
324 324 for inp, expected in pairs:
325 325 out = func(inp)
326 326 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
327 327
328 328
329 329 MyStringIO = StringIO
330 330
331 331 _re_type = type(re.compile(r''))
332 332
333 333 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
334 334 -------
335 335 {2!s}
336 336 -------
337 337 """
338 338
339 339 class AssertPrints(object):
340 340 """Context manager for testing that code prints certain text.
341 341
342 342 Examples
343 343 --------
344 344 >>> with AssertPrints("abc", suppress=False):
345 345 ... print("abcd")
346 346 ... print("def")
347 347 ...
348 348 abcd
349 349 def
350 350 """
351 351 def __init__(self, s, channel='stdout', suppress=True):
352 352 self.s = s
353 353 if isinstance(self.s, (str, _re_type)):
354 354 self.s = [self.s]
355 355 self.channel = channel
356 356 self.suppress = suppress
357 357
358 358 def __enter__(self):
359 359 self.orig_stream = getattr(sys, self.channel)
360 360 self.buffer = MyStringIO()
361 361 self.tee = Tee(self.buffer, channel=self.channel)
362 362 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
363 363
364 364 def __exit__(self, etype, value, traceback):
365 365 try:
366 366 if value is not None:
367 367 # If an error was raised, don't check anything else
368 368 return False
369 369 self.tee.flush()
370 370 setattr(sys, self.channel, self.orig_stream)
371 371 printed = self.buffer.getvalue()
372 372 for s in self.s:
373 373 if isinstance(s, _re_type):
374 374 assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
375 375 else:
376 376 assert s in printed, notprinted_msg.format(s, self.channel, printed)
377 377 return False
378 378 finally:
379 379 self.tee.close()
380 380
381 381 printed_msg = """Found {0!r} in printed output (on {1}):
382 382 -------
383 383 {2!s}
384 384 -------
385 385 """
386 386
387 387 class AssertNotPrints(AssertPrints):
388 388 """Context manager for checking that certain output *isn't* produced.
389 389
390 390 Counterpart of AssertPrints"""
391 391 def __exit__(self, etype, value, traceback):
392 392 try:
393 393 if value is not None:
394 394 # If an error was raised, don't check anything else
395 395 self.tee.close()
396 396 return False
397 397 self.tee.flush()
398 398 setattr(sys, self.channel, self.orig_stream)
399 399 printed = self.buffer.getvalue()
400 400 for s in self.s:
401 401 if isinstance(s, _re_type):
402 402 assert not s.search(printed),printed_msg.format(
403 403 s.pattern, self.channel, printed)
404 404 else:
405 405 assert s not in printed, printed_msg.format(
406 406 s, self.channel, printed)
407 407 return False
408 408 finally:
409 409 self.tee.close()
410 410
411 411 @contextmanager
412 412 def mute_warn():
413 413 from IPython.utils import warn
414 414 save_warn = warn.warn
415 415 warn.warn = lambda *a, **kw: None
416 416 try:
417 417 yield
418 418 finally:
419 419 warn.warn = save_warn
420 420
421 421 @contextmanager
422 422 def make_tempfile(name):
423 423 """ Create an empty, named, temporary file for the duration of the context.
424 424 """
425 f = open(name, 'w')
426 f.close()
425 open(name, 'w').close()
427 426 try:
428 427 yield
429 428 finally:
430 429 os.unlink(name)
431 430
432 431 def fake_input(inputs):
433 432 """Temporarily replace the input() function to return the given values
434 433
435 434 Use as a context manager:
436 435
437 436 with fake_input(['result1', 'result2']):
438 437 ...
439 438
440 439 Values are returned in order. If input() is called again after the last value
441 440 was used, EOFError is raised.
442 441 """
443 442 it = iter(inputs)
444 443 def mock_input(prompt=''):
445 444 try:
446 445 return next(it)
447 446 except StopIteration:
448 447 raise EOFError('No more inputs given')
449 448
450 449 return patch('builtins.input', mock_input)
451 450
452 451 def help_output_test(subcommand=''):
453 452 """test that `ipython [subcommand] -h` works"""
454 453 cmd = get_ipython_cmd() + [subcommand, '-h']
455 454 out, err, rc = get_output_error_code(cmd)
456 455 nt.assert_equal(rc, 0, err)
457 456 nt.assert_not_in("Traceback", err)
458 457 nt.assert_in("Options", out)
459 458 nt.assert_in("--help-all", out)
460 459 return out, err
461 460
462 461
463 462 def help_all_output_test(subcommand=''):
464 463 """test that `ipython [subcommand] --help-all` works"""
465 464 cmd = get_ipython_cmd() + [subcommand, '--help-all']
466 465 out, err, rc = get_output_error_code(cmd)
467 466 nt.assert_equal(rc, 0, err)
468 467 nt.assert_not_in("Traceback", err)
469 468 nt.assert_in("Options", out)
470 469 nt.assert_in("Class", out)
471 470 return out, err
472 471
@@ -1,115 +1,114 b''
1 1 # encoding: utf-8
2 2 """Tests for IPython.utils.module_paths.py"""
3 3
4 4 #-----------------------------------------------------------------------------
5 5 # Copyright (C) 2008-2011 The IPython Development Team
6 6 #
7 7 # Distributed under the terms of the BSD License. The full license is in
8 8 # the file COPYING, distributed as part of this software.
9 9 #-----------------------------------------------------------------------------
10 10
11 11 #-----------------------------------------------------------------------------
12 12 # Imports
13 13 #-----------------------------------------------------------------------------
14 14
15 15
16 16 import os
17 17 import shutil
18 18 import sys
19 19 import tempfile
20 20
21 21 from os.path import join, abspath, split
22 22
23 23 from IPython.testing.tools import make_tempfile
24 24
25 25 import IPython.utils.module_paths as mp
26 26
27 27 import nose.tools as nt
28 28
29 29 env = os.environ
30 30 TEST_FILE_PATH = split(abspath(__file__))[0]
31 31
32 32 TMP_TEST_DIR = tempfile.mkdtemp(suffix='with.dot')
33 33 #
34 34 # Setup/teardown functions/decorators
35 35 #
36 36
37 37 old_syspath = sys.path
38 38
39 39 def make_empty_file(fname):
40 f = open(fname, 'w')
41 f.close()
40 open(fname, 'w').close()
42 41
43 42
44 43 def setup():
45 44 """Setup testenvironment for the module:
46 45
47 46 """
48 47 # Do not mask exceptions here. In particular, catching WindowsError is a
49 48 # problem because that exception is only defined on Windows...
50 49 os.makedirs(join(TMP_TEST_DIR, "xmod"))
51 50 os.makedirs(join(TMP_TEST_DIR, "nomod"))
52 51 make_empty_file(join(TMP_TEST_DIR, "xmod/__init__.py"))
53 52 make_empty_file(join(TMP_TEST_DIR, "xmod/sub.py"))
54 53 make_empty_file(join(TMP_TEST_DIR, "pack.py"))
55 54 make_empty_file(join(TMP_TEST_DIR, "packpyc.pyc"))
56 55 sys.path = [TMP_TEST_DIR]
57 56
58 57 def teardown():
59 58 """Teardown testenvironment for the module:
60 59
61 60 - Remove tempdir
62 61 - restore sys.path
63 62 """
64 63 # Note: we remove the parent test dir, which is the root of all test
65 64 # subdirs we may have created. Use shutil instead of os.removedirs, so
66 65 # that non-empty directories are all recursively removed.
67 66 shutil.rmtree(TMP_TEST_DIR)
68 67 sys.path = old_syspath
69 68
70 69 def test_tempdir():
71 70 """
72 71 Ensure the test are done with a temporary file that have a dot somewhere.
73 72 """
74 73 nt.assert_in('.',TMP_TEST_DIR)
75 74
76 75
77 76 def test_find_mod_1():
78 77 """
79 78 Search for a directory's file path.
80 79 Expected output: a path to that directory's __init__.py file.
81 80 """
82 81 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
83 82 nt.assert_equal(mp.find_mod("xmod"), modpath)
84 83
85 84 def test_find_mod_2():
86 85 """
87 86 Search for a directory's file path.
88 87 Expected output: a path to that directory's __init__.py file.
89 88 TODO: Confirm why this is a duplicate test.
90 89 """
91 90 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
92 91 nt.assert_equal(mp.find_mod("xmod"), modpath)
93 92
94 93 def test_find_mod_3():
95 94 """
96 95 Search for a directory + a filename without its .py extension
97 96 Expected output: full path with .py extension.
98 97 """
99 98 modpath = join(TMP_TEST_DIR, "xmod", "sub.py")
100 99 nt.assert_equal(mp.find_mod("xmod.sub"), modpath)
101 100
102 101 def test_find_mod_4():
103 102 """
104 103 Search for a filename without its .py extension
105 104 Expected output: full path with .py extension
106 105 """
107 106 modpath = join(TMP_TEST_DIR, "pack.py")
108 107 nt.assert_equal(mp.find_mod("pack"), modpath)
109 108
110 109 def test_find_mod_5():
111 110 """
112 111 Search for a filename with a .pyc extension
113 112 Expected output: TODO: do we exclude or include .pyc files?
114 113 """
115 114 nt.assert_equal(mp.find_mod("packpyc"), None)
@@ -1,31 +1,31 b''
1 1 import io
2 2 import os.path
3 3 import nose.tools as nt
4 4
5 5 from IPython.utils import openpy
6 6
7 7 mydir = os.path.dirname(__file__)
8 8 nonascii_path = os.path.join(mydir, '../../core/tests/nonascii.py')
9 9
10 10 def test_detect_encoding():
11 f = open(nonascii_path, 'rb')
12 enc, lines = openpy.detect_encoding(f.readline)
11 with open(nonascii_path, 'rb') as f:
12 enc, lines = openpy.detect_encoding(f.readline)
13 13 nt.assert_equal(enc, 'iso-8859-5')
14 14
15 15 def test_read_file():
16 16 read_specified_enc = io.open(nonascii_path, encoding='iso-8859-5').read()
17 17 read_detected_enc = openpy.read_py_file(nonascii_path, skip_encoding_cookie=False)
18 18 nt.assert_equal(read_detected_enc, read_specified_enc)
19 19 assert u'coding: iso-8859-5' in read_detected_enc
20 20
21 21 read_strip_enc_cookie = openpy.read_py_file(nonascii_path, skip_encoding_cookie=True)
22 22 assert u'coding: iso-8859-5' not in read_strip_enc_cookie
23 23
24 24 def test_source_to_unicode():
25 25 with io.open(nonascii_path, 'rb') as f:
26 26 source_bytes = f.read()
27 27 nt.assert_equal(openpy.source_to_unicode(source_bytes, skip_encoding_cookie=False).splitlines(),
28 28 source_bytes.decode('iso-8859-5').splitlines())
29 29
30 30 source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
31 31 nt.assert_not_in(u'coding: iso-8859-5', source_no_cookie)
@@ -1,455 +1,453 b''
1 1 """Attempt to generate templates for module reference with Sphinx
2 2
3 3 XXX - we exclude extension modules
4 4
5 5 To include extension modules, first identify them as valid in the
6 6 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
7 7
8 8 We get functions and classes by parsing the text of .py files.
9 9 Alternatively we could import the modules for discovery, and we'd have
10 10 to do that for extension modules. This would involve changing the
11 11 ``_parse_module`` method to work via import and introspection, and
12 12 might involve changing ``discover_modules`` (which determines which
13 13 files are modules, and therefore which module URIs will be passed to
14 14 ``_parse_module``).
15 15
16 16 NOTE: this is a modified version of a script originally shipped with the
17 17 PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
18 18 project."""
19 19
20 20
21 21 # Stdlib imports
22 22 import ast
23 23 import inspect
24 24 import os
25 25 import re
26 26 from importlib import import_module
27 27
28 28
29 29 class Obj(object):
30 30 '''Namespace to hold arbitrary information.'''
31 31 def __init__(self, **kwargs):
32 32 for k, v in kwargs.items():
33 33 setattr(self, k, v)
34 34
35 35 class FuncClsScanner(ast.NodeVisitor):
36 36 """Scan a module for top-level functions and classes.
37 37
38 38 Skips objects with an @undoc decorator, or a name starting with '_'.
39 39 """
40 40 def __init__(self):
41 41 ast.NodeVisitor.__init__(self)
42 42 self.classes = []
43 43 self.classes_seen = set()
44 44 self.functions = []
45 45
46 46 @staticmethod
47 47 def has_undoc_decorator(node):
48 48 return any(isinstance(d, ast.Name) and d.id == 'undoc' \
49 49 for d in node.decorator_list)
50 50
51 51 def visit_If(self, node):
52 52 if isinstance(node.test, ast.Compare) \
53 53 and isinstance(node.test.left, ast.Name) \
54 54 and node.test.left.id == '__name__':
55 55 return # Ignore classes defined in "if __name__ == '__main__':"
56 56
57 57 self.generic_visit(node)
58 58
59 59 def visit_FunctionDef(self, node):
60 60 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
61 61 and node.name not in self.functions:
62 62 self.functions.append(node.name)
63 63
64 64 def visit_ClassDef(self, node):
65 65 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
66 66 and node.name not in self.classes_seen:
67 67 cls = Obj(name=node.name)
68 68 cls.has_init = any(isinstance(n, ast.FunctionDef) and \
69 69 n.name=='__init__' for n in node.body)
70 70 self.classes.append(cls)
71 71 self.classes_seen.add(node.name)
72 72
73 73 def scan(self, mod):
74 74 self.visit(mod)
75 75 return self.functions, self.classes
76 76
77 77 # Functions and classes
78 78 class ApiDocWriter(object):
79 79 ''' Class for automatic detection and parsing of API docs
80 80 to Sphinx-parsable reST format'''
81 81
82 82 # only separating first two levels
83 83 rst_section_levels = ['*', '=', '-', '~', '^']
84 84
85 85 def __init__(self,
86 86 package_name,
87 87 rst_extension='.rst',
88 88 package_skip_patterns=None,
89 89 module_skip_patterns=None,
90 90 names_from__all__=None,
91 91 ):
92 92 ''' Initialize package for parsing
93 93
94 94 Parameters
95 95 ----------
96 96 package_name : string
97 97 Name of the top-level package. *package_name* must be the
98 98 name of an importable package
99 99 rst_extension : string, optional
100 100 Extension for reST files, default '.rst'
101 101 package_skip_patterns : None or sequence of {strings, regexps}
102 102 Sequence of strings giving URIs of packages to be excluded
103 103 Operates on the package path, starting at (including) the
104 104 first dot in the package path, after *package_name* - so,
105 105 if *package_name* is ``sphinx``, then ``sphinx.util`` will
106 106 result in ``.util`` being passed for earching by these
107 107 regexps. If is None, gives default. Default is:
108 108 ['\.tests$']
109 109 module_skip_patterns : None or sequence
110 110 Sequence of strings giving URIs of modules to be excluded
111 111 Operates on the module name including preceding URI path,
112 112 back to the first dot after *package_name*. For example
113 113 ``sphinx.util.console`` results in the string to search of
114 114 ``.util.console``
115 115 If is None, gives default. Default is:
116 116 ['\.setup$', '\._']
117 117 names_from__all__ : set, optional
118 118 Modules listed in here will be scanned by doing ``from mod import *``,
119 119 rather than finding function and class definitions by scanning the
120 120 AST. This is intended for API modules which expose things defined in
121 121 other files. Modules listed here must define ``__all__`` to avoid
122 122 exposing everything they import.
123 123 '''
124 124 if package_skip_patterns is None:
125 125 package_skip_patterns = ['\\.tests$']
126 126 if module_skip_patterns is None:
127 127 module_skip_patterns = ['\\.setup$', '\\._']
128 128 self.package_name = package_name
129 129 self.rst_extension = rst_extension
130 130 self.package_skip_patterns = package_skip_patterns
131 131 self.module_skip_patterns = module_skip_patterns
132 132 self.names_from__all__ = names_from__all__ or set()
133 133
134 134 def get_package_name(self):
135 135 return self._package_name
136 136
137 137 def set_package_name(self, package_name):
138 138 ''' Set package_name
139 139
140 140 >>> docwriter = ApiDocWriter('sphinx')
141 141 >>> import sphinx
142 142 >>> docwriter.root_path == sphinx.__path__[0]
143 143 True
144 144 >>> docwriter.package_name = 'docutils'
145 145 >>> import docutils
146 146 >>> docwriter.root_path == docutils.__path__[0]
147 147 True
148 148 '''
149 149 # It's also possible to imagine caching the module parsing here
150 150 self._package_name = package_name
151 151 self.root_module = import_module(package_name)
152 152 self.root_path = self.root_module.__path__[0]
153 153 self.written_modules = None
154 154
155 155 package_name = property(get_package_name, set_package_name, None,
156 156 'get/set package_name')
157 157
158 158 def _uri2path(self, uri):
159 159 ''' Convert uri to absolute filepath
160 160
161 161 Parameters
162 162 ----------
163 163 uri : string
164 164 URI of python module to return path for
165 165
166 166 Returns
167 167 -------
168 168 path : None or string
169 169 Returns None if there is no valid path for this URI
170 170 Otherwise returns absolute file system path for URI
171 171
172 172 Examples
173 173 --------
174 174 >>> docwriter = ApiDocWriter('sphinx')
175 175 >>> import sphinx
176 176 >>> modpath = sphinx.__path__[0]
177 177 >>> res = docwriter._uri2path('sphinx.builder')
178 178 >>> res == os.path.join(modpath, 'builder.py')
179 179 True
180 180 >>> res = docwriter._uri2path('sphinx')
181 181 >>> res == os.path.join(modpath, '__init__.py')
182 182 True
183 183 >>> docwriter._uri2path('sphinx.does_not_exist')
184 184
185 185 '''
186 186 if uri == self.package_name:
187 187 return os.path.join(self.root_path, '__init__.py')
188 188 path = uri.replace('.', os.path.sep)
189 189 path = path.replace(self.package_name + os.path.sep, '')
190 190 path = os.path.join(self.root_path, path)
191 191 # XXX maybe check for extensions as well?
192 192 if os.path.exists(path + '.py'): # file
193 193 path += '.py'
194 194 elif os.path.exists(os.path.join(path, '__init__.py')):
195 195 path = os.path.join(path, '__init__.py')
196 196 else:
197 197 return None
198 198 return path
199 199
200 200 def _path2uri(self, dirpath):
201 201 ''' Convert directory path to uri '''
202 202 relpath = dirpath.replace(self.root_path, self.package_name)
203 203 if relpath.startswith(os.path.sep):
204 204 relpath = relpath[1:]
205 205 return relpath.replace(os.path.sep, '.')
206 206
207 207 def _parse_module(self, uri):
208 208 ''' Parse module defined in *uri* '''
209 209 filename = self._uri2path(uri)
210 210 if filename is None:
211 211 # nothing that we could handle here.
212 212 return ([],[])
213 213 with open(filename, 'rb') as f:
214 214 mod = ast.parse(f.read())
215 215 return FuncClsScanner().scan(mod)
216 216
217 217 def _import_funcs_classes(self, uri):
218 218 """Import * from uri, and separate out functions and classes."""
219 219 ns = {}
220 220 exec('from %s import *' % uri, ns)
221 221 funcs, classes = [], []
222 222 for name, obj in ns.items():
223 223 if inspect.isclass(obj):
224 224 cls = Obj(name=name, has_init='__init__' in obj.__dict__)
225 225 classes.append(cls)
226 226 elif inspect.isfunction(obj):
227 227 funcs.append(name)
228 228
229 229 return sorted(funcs), sorted(classes, key=lambda x: x.name)
230 230
231 231 def find_funcs_classes(self, uri):
232 232 """Find the functions and classes defined in the module ``uri``"""
233 233 if uri in self.names_from__all__:
234 234 # For API modules which expose things defined elsewhere, import them
235 235 return self._import_funcs_classes(uri)
236 236 else:
237 237 # For other modules, scan their AST to see what they define
238 238 return self._parse_module(uri)
239 239
240 240 def generate_api_doc(self, uri):
241 241 '''Make autodoc documentation template string for a module
242 242
243 243 Parameters
244 244 ----------
245 245 uri : string
246 246 python location of module - e.g 'sphinx.builder'
247 247
248 248 Returns
249 249 -------
250 250 S : string
251 251 Contents of API doc
252 252 '''
253 253 # get the names of all classes and functions
254 254 functions, classes = self.find_funcs_classes(uri)
255 255 if not len(functions) and not len(classes):
256 256 #print ('WARNING: Empty -', uri) # dbg
257 257 return ''
258 258
259 259 # Make a shorter version of the uri that omits the package name for
260 260 # titles
261 261 uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
262 262
263 263 ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
264 264
265 265 # Set the chapter title to read 'Module:' for all modules except for the
266 266 # main packages
267 267 if '.' in uri:
268 268 chap_title = 'Module: :mod:`' + uri_short + '`'
269 269 else:
270 270 chap_title = ':mod:`' + uri_short + '`'
271 271 ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
272 272
273 273 ad += '\n.. automodule:: ' + uri + '\n'
274 274 ad += '\n.. currentmodule:: ' + uri + '\n'
275 275
276 276 if classes:
277 277 subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
278 278 ad += '\n'+ subhead + '\n' + \
279 279 self.rst_section_levels[2] * len(subhead) + '\n'
280 280
281 281 for c in classes:
282 282 ad += '\n.. autoclass:: ' + c.name + '\n'
283 283 # must NOT exclude from index to keep cross-refs working
284 284 ad += ' :members:\n' \
285 285 ' :show-inheritance:\n'
286 286 if c.has_init:
287 287 ad += '\n .. automethod:: __init__\n'
288 288
289 289 if functions:
290 290 subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
291 291 ad += '\n'+ subhead + '\n' + \
292 292 self.rst_section_levels[2] * len(subhead) + '\n'
293 293 for f in functions:
294 294 # must NOT exclude from index to keep cross-refs working
295 295 ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
296 296 return ad
297 297
298 298 def _survives_exclude(self, matchstr, match_type):
299 299 ''' Returns True if *matchstr* does not match patterns
300 300
301 301 ``self.package_name`` removed from front of string if present
302 302
303 303 Examples
304 304 --------
305 305 >>> dw = ApiDocWriter('sphinx')
306 306 >>> dw._survives_exclude('sphinx.okpkg', 'package')
307 307 True
308 308 >>> dw.package_skip_patterns.append('^\\.badpkg$')
309 309 >>> dw._survives_exclude('sphinx.badpkg', 'package')
310 310 False
311 311 >>> dw._survives_exclude('sphinx.badpkg', 'module')
312 312 True
313 313 >>> dw._survives_exclude('sphinx.badmod', 'module')
314 314 True
315 315 >>> dw.module_skip_patterns.append('^\\.badmod$')
316 316 >>> dw._survives_exclude('sphinx.badmod', 'module')
317 317 False
318 318 '''
319 319 if match_type == 'module':
320 320 patterns = self.module_skip_patterns
321 321 elif match_type == 'package':
322 322 patterns = self.package_skip_patterns
323 323 else:
324 324 raise ValueError('Cannot interpret match type "%s"'
325 325 % match_type)
326 326 # Match to URI without package name
327 327 L = len(self.package_name)
328 328 if matchstr[:L] == self.package_name:
329 329 matchstr = matchstr[L:]
330 330 for pat in patterns:
331 331 try:
332 332 pat.search
333 333 except AttributeError:
334 334 pat = re.compile(pat)
335 335 if pat.search(matchstr):
336 336 return False
337 337 return True
338 338
339 339 def discover_modules(self):
340 340 ''' Return module sequence discovered from ``self.package_name``
341 341
342 342
343 343 Parameters
344 344 ----------
345 345 None
346 346
347 347 Returns
348 348 -------
349 349 mods : sequence
350 350 Sequence of module names within ``self.package_name``
351 351
352 352 Examples
353 353 --------
354 354 >>> dw = ApiDocWriter('sphinx')
355 355 >>> mods = dw.discover_modules()
356 356 >>> 'sphinx.util' in mods
357 357 True
358 358 >>> dw.package_skip_patterns.append('\.util$')
359 359 >>> 'sphinx.util' in dw.discover_modules()
360 360 False
361 361 >>>
362 362 '''
363 363 modules = [self.package_name]
364 364 # raw directory parsing
365 365 for dirpath, dirnames, filenames in os.walk(self.root_path):
366 366 # Check directory names for packages
367 367 root_uri = self._path2uri(os.path.join(self.root_path,
368 368 dirpath))
369 369 for dirname in dirnames[:]: # copy list - we modify inplace
370 370 package_uri = '.'.join((root_uri, dirname))
371 371 if (self._uri2path(package_uri) and
372 372 self._survives_exclude(package_uri, 'package')):
373 373 modules.append(package_uri)
374 374 else:
375 375 dirnames.remove(dirname)
376 376 # Check filenames for modules
377 377 for filename in filenames:
378 378 module_name = filename[:-3]
379 379 module_uri = '.'.join((root_uri, module_name))
380 380 if (self._uri2path(module_uri) and
381 381 self._survives_exclude(module_uri, 'module')):
382 382 modules.append(module_uri)
383 383 return sorted(modules)
384 384
385 385 def write_modules_api(self, modules,outdir):
386 386 # write the list
387 387 written_modules = []
388 388 for m in modules:
389 389 api_str = self.generate_api_doc(m)
390 390 if not api_str:
391 391 continue
392 392 # write out to file
393 393 outfile = os.path.join(outdir,
394 394 m + self.rst_extension)
395 fileobj = open(outfile, 'wt')
396 fileobj.write(api_str)
397 fileobj.close()
395 with open(outfile, 'wt') as fileobj:
396 fileobj.write(api_str)
398 397 written_modules.append(m)
399 398 self.written_modules = written_modules
400 399
401 400 def write_api_docs(self, outdir):
402 401 """Generate API reST files.
403 402
404 403 Parameters
405 404 ----------
406 405 outdir : string
407 406 Directory name in which to store files
408 407 We create automatic filenames for each module
409 408
410 409 Returns
411 410 -------
412 411 None
413 412
414 413 Notes
415 414 -----
416 415 Sets self.written_modules to list of written modules
417 416 """
418 417 if not os.path.exists(outdir):
419 418 os.mkdir(outdir)
420 419 # compose list of modules
421 420 modules = self.discover_modules()
422 421 self.write_modules_api(modules,outdir)
423 422
424 423 def write_index(self, outdir, path='gen.rst', relative_to=None):
425 424 """Make a reST API index file from written files
426 425
427 426 Parameters
428 427 ----------
429 428 outdir : string
430 429 Directory to which to write generated index file
431 430 path : string
432 431 Filename to write index to
433 432 relative_to : string
434 433 path to which written filenames are relative. This
435 434 component of the written file path will be removed from
436 435 outdir, in the generated index. Default is None, meaning,
437 436 leave path as it is.
438 437 """
439 438 if self.written_modules is None:
440 439 raise ValueError('No modules written')
441 440 # Get full filename path
442 441 path = os.path.join(outdir, path)
443 442 # Path written into index is relative to rootpath
444 443 if relative_to is not None:
445 444 relpath = outdir.replace(relative_to + os.path.sep, '')
446 445 else:
447 446 relpath = outdir
448 idx = open(path,'wt')
449 w = idx.write
450 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
451 w('.. autosummary::\n'
452 ' :toctree: %s\n\n' % relpath)
453 for mod in self.written_modules:
454 w(' %s\n' % mod)
455 idx.close()
447 with open(path,'wt') as idx:
448 w = idx.write
449 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
450 w('.. autosummary::\n'
451 ' :toctree: %s\n\n' % relpath)
452 for mod in self.written_modules:
453 w(' %s\n' % mod)
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now