##// END OF EJS Templates
Fix ResourceWarning: unclosed file...
Mickaël Schoentgen -
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -1,225 +1,226 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """sys.excepthook for IPython itself, leaves a detailed report on disk.
2 """sys.excepthook for IPython itself, leaves a detailed report on disk.
3
3
4 Authors:
4 Authors:
5
5
6 * Fernando Perez
6 * Fernando Perez
7 * Brian E. Granger
7 * Brian E. Granger
8 """
8 """
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
11 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
12 # Copyright (C) 2008-2011 The IPython Development Team
12 # Copyright (C) 2008-2011 The IPython Development Team
13 #
13 #
14 # Distributed under the terms of the BSD License. The full license is in
14 # Distributed under the terms of the BSD License. The full license is in
15 # the file COPYING, distributed as part of this software.
15 # the file COPYING, distributed as part of this software.
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Imports
19 # Imports
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21
21
22 import os
22 import os
23 import sys
23 import sys
24 import traceback
24 import traceback
25 from pprint import pformat
25 from pprint import pformat
26
26
27 from IPython.core import ultratb
27 from IPython.core import ultratb
28 from IPython.core.release import author_email
28 from IPython.core.release import author_email
29 from IPython.utils.sysinfo import sys_info
29 from IPython.utils.sysinfo import sys_info
30 from IPython.utils.py3compat import input
30 from IPython.utils.py3compat import input
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Code
33 # Code
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 # Template for the user message.
36 # Template for the user message.
37 _default_message_template = """\
37 _default_message_template = """\
38 Oops, {app_name} crashed. We do our best to make it stable, but...
38 Oops, {app_name} crashed. We do our best to make it stable, but...
39
39
40 A crash report was automatically generated with the following information:
40 A crash report was automatically generated with the following information:
41 - A verbatim copy of the crash traceback.
41 - A verbatim copy of the crash traceback.
42 - A copy of your input history during this session.
42 - A copy of your input history during this session.
43 - Data on your current {app_name} configuration.
43 - Data on your current {app_name} configuration.
44
44
45 It was left in the file named:
45 It was left in the file named:
46 \t'{crash_report_fname}'
46 \t'{crash_report_fname}'
47 If you can email this file to the developers, the information in it will help
47 If you can email this file to the developers, the information in it will help
48 them in understanding and correcting the problem.
48 them in understanding and correcting the problem.
49
49
50 You can mail it to: {contact_name} at {contact_email}
50 You can mail it to: {contact_name} at {contact_email}
51 with the subject '{app_name} Crash Report'.
51 with the subject '{app_name} Crash Report'.
52
52
53 If you want to do it now, the following command will work (under Unix):
53 If you want to do it now, the following command will work (under Unix):
54 mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
54 mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
55
55
56 In your email, please also include information about:
56 In your email, please also include information about:
57 - The operating system under which the crash happened: Linux, macOS, Windows,
57 - The operating system under which the crash happened: Linux, macOS, Windows,
58 other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
58 other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
59 Windows 10 Pro), and whether it is 32-bit or 64-bit;
59 Windows 10 Pro), and whether it is 32-bit or 64-bit;
60 - How {app_name} was installed: using pip or conda, from GitHub, as part of
60 - How {app_name} was installed: using pip or conda, from GitHub, as part of
61 a Docker container, or other, providing more detail if possible;
61 a Docker container, or other, providing more detail if possible;
62 - How to reproduce the crash: what exact sequence of instructions can one
62 - How to reproduce the crash: what exact sequence of instructions can one
63 input to get the same crash? Ideally, find a minimal yet complete sequence
63 input to get the same crash? Ideally, find a minimal yet complete sequence
64 of instructions that yields the crash.
64 of instructions that yields the crash.
65
65
66 To ensure accurate tracking of this issue, please file a report about it at:
66 To ensure accurate tracking of this issue, please file a report about it at:
67 {bug_tracker}
67 {bug_tracker}
68 """
68 """
69
69
70 _lite_message_template = """
70 _lite_message_template = """
71 If you suspect this is an IPython bug, please report it at:
71 If you suspect this is an IPython bug, please report it at:
72 https://github.com/ipython/ipython/issues
72 https://github.com/ipython/ipython/issues
73 or send an email to the mailing list at {email}
73 or send an email to the mailing list at {email}
74
74
75 You can print a more detailed traceback right now with "%tb", or use "%debug"
75 You can print a more detailed traceback right now with "%tb", or use "%debug"
76 to interactively debug it.
76 to interactively debug it.
77
77
78 Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
78 Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
79 {config}Application.verbose_crash=True
79 {config}Application.verbose_crash=True
80 """
80 """
81
81
82
82
83 class CrashHandler(object):
83 class CrashHandler(object):
84 """Customizable crash handlers for IPython applications.
84 """Customizable crash handlers for IPython applications.
85
85
86 Instances of this class provide a :meth:`__call__` method which can be
86 Instances of this class provide a :meth:`__call__` method which can be
87 used as a ``sys.excepthook``. The :meth:`__call__` signature is::
87 used as a ``sys.excepthook``. The :meth:`__call__` signature is::
88
88
89 def __call__(self, etype, evalue, etb)
89 def __call__(self, etype, evalue, etb)
90 """
90 """
91
91
92 message_template = _default_message_template
92 message_template = _default_message_template
93 section_sep = '\n\n'+'*'*75+'\n\n'
93 section_sep = '\n\n'+'*'*75+'\n\n'
94
94
95 def __init__(self, app, contact_name=None, contact_email=None,
95 def __init__(self, app, contact_name=None, contact_email=None,
96 bug_tracker=None, show_crash_traceback=True, call_pdb=False):
96 bug_tracker=None, show_crash_traceback=True, call_pdb=False):
97 """Create a new crash handler
97 """Create a new crash handler
98
98
99 Parameters
99 Parameters
100 ----------
100 ----------
101 app : Application
101 app : Application
102 A running :class:`Application` instance, which will be queried at
102 A running :class:`Application` instance, which will be queried at
103 crash time for internal information.
103 crash time for internal information.
104
104
105 contact_name : str
105 contact_name : str
106 A string with the name of the person to contact.
106 A string with the name of the person to contact.
107
107
108 contact_email : str
108 contact_email : str
109 A string with the email address of the contact.
109 A string with the email address of the contact.
110
110
111 bug_tracker : str
111 bug_tracker : str
112 A string with the URL for your project's bug tracker.
112 A string with the URL for your project's bug tracker.
113
113
114 show_crash_traceback : bool
114 show_crash_traceback : bool
115 If false, don't print the crash traceback on stderr, only generate
115 If false, don't print the crash traceback on stderr, only generate
116 the on-disk report
116 the on-disk report
117
117
118 Non-argument instance attributes:
118 Non-argument instance attributes:
119
119
120 These instances contain some non-argument attributes which allow for
120 These instances contain some non-argument attributes which allow for
121 further customization of the crash handler's behavior. Please see the
121 further customization of the crash handler's behavior. Please see the
122 source for further details.
122 source for further details.
123 """
123 """
124 self.crash_report_fname = "Crash_report_%s.txt" % app.name
124 self.crash_report_fname = "Crash_report_%s.txt" % app.name
125 self.app = app
125 self.app = app
126 self.call_pdb = call_pdb
126 self.call_pdb = call_pdb
127 #self.call_pdb = True # dbg
127 #self.call_pdb = True # dbg
128 self.show_crash_traceback = show_crash_traceback
128 self.show_crash_traceback = show_crash_traceback
129 self.info = dict(app_name = app.name,
129 self.info = dict(app_name = app.name,
130 contact_name = contact_name,
130 contact_name = contact_name,
131 contact_email = contact_email,
131 contact_email = contact_email,
132 bug_tracker = bug_tracker,
132 bug_tracker = bug_tracker,
133 crash_report_fname = self.crash_report_fname)
133 crash_report_fname = self.crash_report_fname)
134
134
135
135
136 def __call__(self, etype, evalue, etb):
136 def __call__(self, etype, evalue, etb):
137 """Handle an exception, call for compatible with sys.excepthook"""
137 """Handle an exception, call for compatible with sys.excepthook"""
138
138
139 # do not allow the crash handler to be called twice without reinstalling it
139 # do not allow the crash handler to be called twice without reinstalling it
140 # this prevents unlikely errors in the crash handling from entering an
140 # this prevents unlikely errors in the crash handling from entering an
141 # infinite loop.
141 # infinite loop.
142 sys.excepthook = sys.__excepthook__
142 sys.excepthook = sys.__excepthook__
143
143
144 # Report tracebacks shouldn't use color in general (safer for users)
144 # Report tracebacks shouldn't use color in general (safer for users)
145 color_scheme = 'NoColor'
145 color_scheme = 'NoColor'
146
146
147 # Use this ONLY for developer debugging (keep commented out for release)
147 # Use this ONLY for developer debugging (keep commented out for release)
148 #color_scheme = 'Linux' # dbg
148 #color_scheme = 'Linux' # dbg
149 try:
149 try:
150 rptdir = self.app.ipython_dir
150 rptdir = self.app.ipython_dir
151 except:
151 except:
152 rptdir = os.getcwd()
152 rptdir = os.getcwd()
153 if rptdir is None or not os.path.isdir(rptdir):
153 if rptdir is None or not os.path.isdir(rptdir):
154 rptdir = os.getcwd()
154 rptdir = os.getcwd()
155 report_name = os.path.join(rptdir,self.crash_report_fname)
155 report_name = os.path.join(rptdir,self.crash_report_fname)
156 # write the report filename into the instance dict so it can get
156 # write the report filename into the instance dict so it can get
157 # properly expanded out in the user message template
157 # properly expanded out in the user message template
158 self.crash_report_fname = report_name
158 self.crash_report_fname = report_name
159 self.info['crash_report_fname'] = report_name
159 self.info['crash_report_fname'] = report_name
160 TBhandler = ultratb.VerboseTB(
160 TBhandler = ultratb.VerboseTB(
161 color_scheme=color_scheme,
161 color_scheme=color_scheme,
162 long_header=1,
162 long_header=1,
163 call_pdb=self.call_pdb,
163 call_pdb=self.call_pdb,
164 )
164 )
165 if self.call_pdb:
165 if self.call_pdb:
166 TBhandler(etype,evalue,etb)
166 TBhandler(etype,evalue,etb)
167 return
167 return
168 else:
168 else:
169 traceback = TBhandler.text(etype,evalue,etb,context=31)
169 traceback = TBhandler.text(etype,evalue,etb,context=31)
170
170
171 # print traceback to screen
171 # print traceback to screen
172 if self.show_crash_traceback:
172 if self.show_crash_traceback:
173 print(traceback, file=sys.stderr)
173 print(traceback, file=sys.stderr)
174
174
175 # and generate a complete report on disk
175 # and generate a complete report on disk
176 try:
176 try:
177 report = open(report_name,'w')
177 report = open(report_name,'w')
178 except:
178 except:
179 print('Could not create crash report on disk.', file=sys.stderr)
179 print('Could not create crash report on disk.', file=sys.stderr)
180 return
180 return
181
181
182 with report:
182 # Inform user on stderr of what happened
183 # Inform user on stderr of what happened
183 print('\n'+'*'*70+'\n', file=sys.stderr)
184 print('\n'+'*'*70+'\n', file=sys.stderr)
184 print(self.message_template.format(**self.info), file=sys.stderr)
185 print(self.message_template.format(**self.info), file=sys.stderr)
185
186
186 # Construct report on disk
187 # Construct report on disk
187 report.write(self.make_report(traceback))
188 report.write(self.make_report(traceback))
188 report.close()
189
189 input("Hit <Enter> to quit (your terminal may close):")
190 input("Hit <Enter> to quit (your terminal may close):")
190
191
191 def make_report(self,traceback):
192 def make_report(self,traceback):
192 """Return a string containing a crash report."""
193 """Return a string containing a crash report."""
193
194
194 sec_sep = self.section_sep
195 sec_sep = self.section_sep
195
196
196 report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
197 report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
197 rpt_add = report.append
198 rpt_add = report.append
198 rpt_add(sys_info())
199 rpt_add(sys_info())
199
200
200 try:
201 try:
201 config = pformat(self.app.config)
202 config = pformat(self.app.config)
202 rpt_add(sec_sep)
203 rpt_add(sec_sep)
203 rpt_add('Application name: %s\n\n' % self.app_name)
204 rpt_add('Application name: %s\n\n' % self.app_name)
204 rpt_add('Current user configuration structure:\n\n')
205 rpt_add('Current user configuration structure:\n\n')
205 rpt_add(config)
206 rpt_add(config)
206 except:
207 except:
207 pass
208 pass
208 rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
209 rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
209
210
210 return ''.join(report)
211 return ''.join(report)
211
212
212
213
213 def crash_handler_lite(etype, evalue, tb):
214 def crash_handler_lite(etype, evalue, tb):
214 """a light excepthook, adding a small message to the usual traceback"""
215 """a light excepthook, adding a small message to the usual traceback"""
215 traceback.print_exception(etype, evalue, tb)
216 traceback.print_exception(etype, evalue, tb)
216
217
217 from IPython.core.interactiveshell import InteractiveShell
218 from IPython.core.interactiveshell import InteractiveShell
218 if InteractiveShell.initialized():
219 if InteractiveShell.initialized():
219 # we are in a Shell environment, give %magic example
220 # we are in a Shell environment, give %magic example
220 config = "%config "
221 config = "%config "
221 else:
222 else:
222 # we are not in a shell, show generic config
223 # we are not in a shell, show generic config
223 config = "c."
224 config = "c."
224 print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
225 print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
225
226
@@ -1,645 +1,644 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 Pdb debugger class.
3 Pdb debugger class.
4
4
5 Modified from the standard pdb.Pdb class to avoid including readline, so that
5 Modified from the standard pdb.Pdb class to avoid including readline, so that
6 the command line completion of other programs which include this isn't
6 the command line completion of other programs which include this isn't
7 damaged.
7 damaged.
8
8
9 In the future, this class will be expanded with improvements over the standard
9 In the future, this class will be expanded with improvements over the standard
10 pdb.
10 pdb.
11
11
12 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
12 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
13 changes. Licensing should therefore be under the standard Python terms. For
13 changes. Licensing should therefore be under the standard Python terms. For
14 details on the PSF (Python Software Foundation) standard license, see:
14 details on the PSF (Python Software Foundation) standard license, see:
15
15
16 https://docs.python.org/2/license.html
16 https://docs.python.org/2/license.html
17 """
17 """
18
18
19 #*****************************************************************************
19 #*****************************************************************************
20 #
20 #
21 # This file is licensed under the PSF license.
21 # This file is licensed under the PSF license.
22 #
22 #
23 # Copyright (C) 2001 Python Software Foundation, www.python.org
23 # Copyright (C) 2001 Python Software Foundation, www.python.org
24 # Copyright (C) 2005-2006 Fernando Perez. <fperez@colorado.edu>
24 # Copyright (C) 2005-2006 Fernando Perez. <fperez@colorado.edu>
25 #
25 #
26 #
26 #
27 #*****************************************************************************
27 #*****************************************************************************
28
28
29 import bdb
29 import bdb
30 import functools
30 import functools
31 import inspect
31 import inspect
32 import linecache
32 import linecache
33 import sys
33 import sys
34 import warnings
34 import warnings
35 import re
35 import re
36
36
37 from IPython import get_ipython
37 from IPython import get_ipython
38 from IPython.utils import PyColorize
38 from IPython.utils import PyColorize
39 from IPython.utils import coloransi, py3compat
39 from IPython.utils import coloransi, py3compat
40 from IPython.core.excolors import exception_colors
40 from IPython.core.excolors import exception_colors
41 from IPython.testing.skipdoctest import skip_doctest
41 from IPython.testing.skipdoctest import skip_doctest
42
42
43
43
44 prompt = 'ipdb> '
44 prompt = 'ipdb> '
45
45
46 #We have to check this directly from sys.argv, config struct not yet available
46 #We have to check this directly from sys.argv, config struct not yet available
47 from pdb import Pdb as OldPdb
47 from pdb import Pdb as OldPdb
48
48
49 # Allow the set_trace code to operate outside of an ipython instance, even if
49 # Allow the set_trace code to operate outside of an ipython instance, even if
50 # it does so with some limitations. The rest of this support is implemented in
50 # it does so with some limitations. The rest of this support is implemented in
51 # the Tracer constructor.
51 # the Tracer constructor.
52
52
53 def make_arrow(pad):
53 def make_arrow(pad):
54 """generate the leading arrow in front of traceback or debugger"""
54 """generate the leading arrow in front of traceback or debugger"""
55 if pad >= 2:
55 if pad >= 2:
56 return '-'*(pad-2) + '> '
56 return '-'*(pad-2) + '> '
57 elif pad == 1:
57 elif pad == 1:
58 return '>'
58 return '>'
59 return ''
59 return ''
60
60
61
61
62 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
62 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
63 """Exception hook which handles `BdbQuit` exceptions.
63 """Exception hook which handles `BdbQuit` exceptions.
64
64
65 All other exceptions are processed using the `excepthook`
65 All other exceptions are processed using the `excepthook`
66 parameter.
66 parameter.
67 """
67 """
68 warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
68 warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
69 DeprecationWarning, stacklevel=2)
69 DeprecationWarning, stacklevel=2)
70 if et==bdb.BdbQuit:
70 if et==bdb.BdbQuit:
71 print('Exiting Debugger.')
71 print('Exiting Debugger.')
72 elif excepthook is not None:
72 elif excepthook is not None:
73 excepthook(et, ev, tb)
73 excepthook(et, ev, tb)
74 else:
74 else:
75 # Backwards compatibility. Raise deprecation warning?
75 # Backwards compatibility. Raise deprecation warning?
76 BdbQuit_excepthook.excepthook_ori(et,ev,tb)
76 BdbQuit_excepthook.excepthook_ori(et,ev,tb)
77
77
78
78
79 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
79 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
80 warnings.warn(
80 warnings.warn(
81 "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
81 "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
82 DeprecationWarning, stacklevel=2)
82 DeprecationWarning, stacklevel=2)
83 print('Exiting Debugger.')
83 print('Exiting Debugger.')
84
84
85
85
86 class Tracer(object):
86 class Tracer(object):
87 """
87 """
88 DEPRECATED
88 DEPRECATED
89
89
90 Class for local debugging, similar to pdb.set_trace.
90 Class for local debugging, similar to pdb.set_trace.
91
91
92 Instances of this class, when called, behave like pdb.set_trace, but
92 Instances of this class, when called, behave like pdb.set_trace, but
93 providing IPython's enhanced capabilities.
93 providing IPython's enhanced capabilities.
94
94
95 This is implemented as a class which must be initialized in your own code
95 This is implemented as a class which must be initialized in your own code
96 and not as a standalone function because we need to detect at runtime
96 and not as a standalone function because we need to detect at runtime
97 whether IPython is already active or not. That detection is done in the
97 whether IPython is already active or not. That detection is done in the
98 constructor, ensuring that this code plays nicely with a running IPython,
98 constructor, ensuring that this code plays nicely with a running IPython,
99 while functioning acceptably (though with limitations) if outside of it.
99 while functioning acceptably (though with limitations) if outside of it.
100 """
100 """
101
101
102 @skip_doctest
102 @skip_doctest
103 def __init__(self, colors=None):
103 def __init__(self, colors=None):
104 """
104 """
105 DEPRECATED
105 DEPRECATED
106
106
107 Create a local debugger instance.
107 Create a local debugger instance.
108
108
109 Parameters
109 Parameters
110 ----------
110 ----------
111
111
112 colors : str, optional
112 colors : str, optional
113 The name of the color scheme to use, it must be one of IPython's
113 The name of the color scheme to use, it must be one of IPython's
114 valid color schemes. If not given, the function will default to
114 valid color schemes. If not given, the function will default to
115 the current IPython scheme when running inside IPython, and to
115 the current IPython scheme when running inside IPython, and to
116 'NoColor' otherwise.
116 'NoColor' otherwise.
117
117
118 Examples
118 Examples
119 --------
119 --------
120 ::
120 ::
121
121
122 from IPython.core.debugger import Tracer; debug_here = Tracer()
122 from IPython.core.debugger import Tracer; debug_here = Tracer()
123
123
124 Later in your code::
124 Later in your code::
125
125
126 debug_here() # -> will open up the debugger at that point.
126 debug_here() # -> will open up the debugger at that point.
127
127
128 Once the debugger activates, you can use all of its regular commands to
128 Once the debugger activates, you can use all of its regular commands to
129 step through code, set breakpoints, etc. See the pdb documentation
129 step through code, set breakpoints, etc. See the pdb documentation
130 from the Python standard library for usage details.
130 from the Python standard library for usage details.
131 """
131 """
132 warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
132 warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
133 "`IPython.core.debugger.Pdb.set_trace()`",
133 "`IPython.core.debugger.Pdb.set_trace()`",
134 DeprecationWarning, stacklevel=2)
134 DeprecationWarning, stacklevel=2)
135
135
136 ip = get_ipython()
136 ip = get_ipython()
137 if ip is None:
137 if ip is None:
138 # Outside of ipython, we set our own exception hook manually
138 # Outside of ipython, we set our own exception hook manually
139 sys.excepthook = functools.partial(BdbQuit_excepthook,
139 sys.excepthook = functools.partial(BdbQuit_excepthook,
140 excepthook=sys.excepthook)
140 excepthook=sys.excepthook)
141 def_colors = 'NoColor'
141 def_colors = 'NoColor'
142 else:
142 else:
143 # In ipython, we use its custom exception handler mechanism
143 # In ipython, we use its custom exception handler mechanism
144 def_colors = ip.colors
144 def_colors = ip.colors
145 ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
145 ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
146
146
147 if colors is None:
147 if colors is None:
148 colors = def_colors
148 colors = def_colors
149
149
150 # The stdlib debugger internally uses a modified repr from the `repr`
150 # The stdlib debugger internally uses a modified repr from the `repr`
151 # module, that limits the length of printed strings to a hardcoded
151 # module, that limits the length of printed strings to a hardcoded
152 # limit of 30 characters. That much trimming is too aggressive, let's
152 # limit of 30 characters. That much trimming is too aggressive, let's
153 # at least raise that limit to 80 chars, which should be enough for
153 # at least raise that limit to 80 chars, which should be enough for
154 # most interactive uses.
154 # most interactive uses.
155 try:
155 try:
156 try:
156 try:
157 from reprlib import aRepr # Py 3
157 from reprlib import aRepr # Py 3
158 except ImportError:
158 except ImportError:
159 from repr import aRepr # Py 2
159 from repr import aRepr # Py 2
160 aRepr.maxstring = 80
160 aRepr.maxstring = 80
161 except:
161 except:
162 # This is only a user-facing convenience, so any error we encounter
162 # This is only a user-facing convenience, so any error we encounter
163 # here can be warned about but can be otherwise ignored. These
163 # here can be warned about but can be otherwise ignored. These
164 # printouts will tell us about problems if this API changes
164 # printouts will tell us about problems if this API changes
165 import traceback
165 import traceback
166 traceback.print_exc()
166 traceback.print_exc()
167
167
168 self.debugger = Pdb(colors)
168 self.debugger = Pdb(colors)
169
169
170 def __call__(self):
170 def __call__(self):
171 """Starts an interactive debugger at the point where called.
171 """Starts an interactive debugger at the point where called.
172
172
173 This is similar to the pdb.set_trace() function from the std lib, but
173 This is similar to the pdb.set_trace() function from the std lib, but
174 using IPython's enhanced debugger."""
174 using IPython's enhanced debugger."""
175
175
176 self.debugger.set_trace(sys._getframe().f_back)
176 self.debugger.set_trace(sys._getframe().f_back)
177
177
178
178
179 RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
179 RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
180
180
181
181
182 def strip_indentation(multiline_string):
182 def strip_indentation(multiline_string):
183 return RGX_EXTRA_INDENT.sub('', multiline_string)
183 return RGX_EXTRA_INDENT.sub('', multiline_string)
184
184
185
185
186 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
186 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
187 """Make new_fn have old_fn's doc string. This is particularly useful
187 """Make new_fn have old_fn's doc string. This is particularly useful
188 for the ``do_...`` commands that hook into the help system.
188 for the ``do_...`` commands that hook into the help system.
189 Adapted from from a comp.lang.python posting
189 Adapted from from a comp.lang.python posting
190 by Duncan Booth."""
190 by Duncan Booth."""
191 def wrapper(*args, **kw):
191 def wrapper(*args, **kw):
192 return new_fn(*args, **kw)
192 return new_fn(*args, **kw)
193 if old_fn.__doc__:
193 if old_fn.__doc__:
194 wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
194 wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
195 return wrapper
195 return wrapper
196
196
197
197
198 def _file_lines(fname):
198 def _file_lines(fname):
199 """Return the contents of a named file as a list of lines.
199 """Return the contents of a named file as a list of lines.
200
200
201 This function never raises an IOError exception: if the file can't be
201 This function never raises an IOError exception: if the file can't be
202 read, it simply returns an empty list."""
202 read, it simply returns an empty list."""
203
203
204 try:
204 try:
205 outfile = open(fname)
205 outfile = open(fname)
206 except IOError:
206 except IOError:
207 return []
207 return []
208 else:
208 else:
209 out = outfile.readlines()
209 with out:
210 outfile.close()
210 return outfile.readlines()
211 return out
212
211
213
212
214 class Pdb(OldPdb):
213 class Pdb(OldPdb):
215 """Modified Pdb class, does not load readline.
214 """Modified Pdb class, does not load readline.
216
215
217 for a standalone version that uses prompt_toolkit, see
216 for a standalone version that uses prompt_toolkit, see
218 `IPython.terminal.debugger.TerminalPdb` and
217 `IPython.terminal.debugger.TerminalPdb` and
219 `IPython.terminal.debugger.set_trace()`
218 `IPython.terminal.debugger.set_trace()`
220 """
219 """
221
220
222 def __init__(self, color_scheme=None, completekey=None,
221 def __init__(self, color_scheme=None, completekey=None,
223 stdin=None, stdout=None, context=5):
222 stdin=None, stdout=None, context=5):
224
223
225 # Parent constructor:
224 # Parent constructor:
226 try:
225 try:
227 self.context = int(context)
226 self.context = int(context)
228 if self.context <= 0:
227 if self.context <= 0:
229 raise ValueError("Context must be a positive integer")
228 raise ValueError("Context must be a positive integer")
230 except (TypeError, ValueError):
229 except (TypeError, ValueError):
231 raise ValueError("Context must be a positive integer")
230 raise ValueError("Context must be a positive integer")
232
231
233 OldPdb.__init__(self, completekey, stdin, stdout)
232 OldPdb.__init__(self, completekey, stdin, stdout)
234
233
235 # IPython changes...
234 # IPython changes...
236 self.shell = get_ipython()
235 self.shell = get_ipython()
237
236
238 if self.shell is None:
237 if self.shell is None:
239 save_main = sys.modules['__main__']
238 save_main = sys.modules['__main__']
240 # No IPython instance running, we must create one
239 # No IPython instance running, we must create one
241 from IPython.terminal.interactiveshell import \
240 from IPython.terminal.interactiveshell import \
242 TerminalInteractiveShell
241 TerminalInteractiveShell
243 self.shell = TerminalInteractiveShell.instance()
242 self.shell = TerminalInteractiveShell.instance()
244 # needed by any code which calls __import__("__main__") after
243 # needed by any code which calls __import__("__main__") after
245 # the debugger was entered. See also #9941.
244 # the debugger was entered. See also #9941.
246 sys.modules['__main__'] = save_main
245 sys.modules['__main__'] = save_main
247
246
248 if color_scheme is not None:
247 if color_scheme is not None:
249 warnings.warn(
248 warnings.warn(
250 "The `color_scheme` argument is deprecated since version 5.1",
249 "The `color_scheme` argument is deprecated since version 5.1",
251 DeprecationWarning, stacklevel=2)
250 DeprecationWarning, stacklevel=2)
252 else:
251 else:
253 color_scheme = self.shell.colors
252 color_scheme = self.shell.colors
254
253
255 self.aliases = {}
254 self.aliases = {}
256
255
257 # Create color table: we copy the default one from the traceback
256 # Create color table: we copy the default one from the traceback
258 # module and add a few attributes needed for debugging
257 # module and add a few attributes needed for debugging
259 self.color_scheme_table = exception_colors()
258 self.color_scheme_table = exception_colors()
260
259
261 # shorthands
260 # shorthands
262 C = coloransi.TermColors
261 C = coloransi.TermColors
263 cst = self.color_scheme_table
262 cst = self.color_scheme_table
264
263
265 cst['NoColor'].colors.prompt = C.NoColor
264 cst['NoColor'].colors.prompt = C.NoColor
266 cst['NoColor'].colors.breakpoint_enabled = C.NoColor
265 cst['NoColor'].colors.breakpoint_enabled = C.NoColor
267 cst['NoColor'].colors.breakpoint_disabled = C.NoColor
266 cst['NoColor'].colors.breakpoint_disabled = C.NoColor
268
267
269 cst['Linux'].colors.prompt = C.Green
268 cst['Linux'].colors.prompt = C.Green
270 cst['Linux'].colors.breakpoint_enabled = C.LightRed
269 cst['Linux'].colors.breakpoint_enabled = C.LightRed
271 cst['Linux'].colors.breakpoint_disabled = C.Red
270 cst['Linux'].colors.breakpoint_disabled = C.Red
272
271
273 cst['LightBG'].colors.prompt = C.Blue
272 cst['LightBG'].colors.prompt = C.Blue
274 cst['LightBG'].colors.breakpoint_enabled = C.LightRed
273 cst['LightBG'].colors.breakpoint_enabled = C.LightRed
275 cst['LightBG'].colors.breakpoint_disabled = C.Red
274 cst['LightBG'].colors.breakpoint_disabled = C.Red
276
275
277 cst['Neutral'].colors.prompt = C.Blue
276 cst['Neutral'].colors.prompt = C.Blue
278 cst['Neutral'].colors.breakpoint_enabled = C.LightRed
277 cst['Neutral'].colors.breakpoint_enabled = C.LightRed
279 cst['Neutral'].colors.breakpoint_disabled = C.Red
278 cst['Neutral'].colors.breakpoint_disabled = C.Red
280
279
281
280
282 # Add a python parser so we can syntax highlight source while
281 # Add a python parser so we can syntax highlight source while
283 # debugging.
282 # debugging.
284 self.parser = PyColorize.Parser(style=color_scheme)
283 self.parser = PyColorize.Parser(style=color_scheme)
285 self.set_colors(color_scheme)
284 self.set_colors(color_scheme)
286
285
287 # Set the prompt - the default prompt is '(Pdb)'
286 # Set the prompt - the default prompt is '(Pdb)'
288 self.prompt = prompt
287 self.prompt = prompt
289
288
290 def set_colors(self, scheme):
289 def set_colors(self, scheme):
291 """Shorthand access to the color table scheme selector method."""
290 """Shorthand access to the color table scheme selector method."""
292 self.color_scheme_table.set_active_scheme(scheme)
291 self.color_scheme_table.set_active_scheme(scheme)
293 self.parser.style = scheme
292 self.parser.style = scheme
294
293
295 def interaction(self, frame, traceback):
294 def interaction(self, frame, traceback):
296 try:
295 try:
297 OldPdb.interaction(self, frame, traceback)
296 OldPdb.interaction(self, frame, traceback)
298 except KeyboardInterrupt:
297 except KeyboardInterrupt:
299 sys.stdout.write('\n' + self.shell.get_exception_only())
298 sys.stdout.write('\n' + self.shell.get_exception_only())
300
299
301 def new_do_up(self, arg):
300 def new_do_up(self, arg):
302 OldPdb.do_up(self, arg)
301 OldPdb.do_up(self, arg)
303 do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
302 do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
304
303
305 def new_do_down(self, arg):
304 def new_do_down(self, arg):
306 OldPdb.do_down(self, arg)
305 OldPdb.do_down(self, arg)
307
306
308 do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
307 do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
309
308
310 def new_do_frame(self, arg):
309 def new_do_frame(self, arg):
311 OldPdb.do_frame(self, arg)
310 OldPdb.do_frame(self, arg)
312
311
313 def new_do_quit(self, arg):
312 def new_do_quit(self, arg):
314
313
315 if hasattr(self, 'old_all_completions'):
314 if hasattr(self, 'old_all_completions'):
316 self.shell.Completer.all_completions=self.old_all_completions
315 self.shell.Completer.all_completions=self.old_all_completions
317
316
318 return OldPdb.do_quit(self, arg)
317 return OldPdb.do_quit(self, arg)
319
318
320 do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
319 do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
321
320
322 def new_do_restart(self, arg):
321 def new_do_restart(self, arg):
323 """Restart command. In the context of ipython this is exactly the same
322 """Restart command. In the context of ipython this is exactly the same
324 thing as 'quit'."""
323 thing as 'quit'."""
325 self.msg("Restart doesn't make sense here. Using 'quit' instead.")
324 self.msg("Restart doesn't make sense here. Using 'quit' instead.")
326 return self.do_quit(arg)
325 return self.do_quit(arg)
327
326
328 def print_stack_trace(self, context=None):
327 def print_stack_trace(self, context=None):
329 if context is None:
328 if context is None:
330 context = self.context
329 context = self.context
331 try:
330 try:
332 context=int(context)
331 context=int(context)
333 if context <= 0:
332 if context <= 0:
334 raise ValueError("Context must be a positive integer")
333 raise ValueError("Context must be a positive integer")
335 except (TypeError, ValueError):
334 except (TypeError, ValueError):
336 raise ValueError("Context must be a positive integer")
335 raise ValueError("Context must be a positive integer")
337 try:
336 try:
338 for frame_lineno in self.stack:
337 for frame_lineno in self.stack:
339 self.print_stack_entry(frame_lineno, context=context)
338 self.print_stack_entry(frame_lineno, context=context)
340 except KeyboardInterrupt:
339 except KeyboardInterrupt:
341 pass
340 pass
342
341
343 def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
342 def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
344 context=None):
343 context=None):
345 if context is None:
344 if context is None:
346 context = self.context
345 context = self.context
347 try:
346 try:
348 context=int(context)
347 context=int(context)
349 if context <= 0:
348 if context <= 0:
350 raise ValueError("Context must be a positive integer")
349 raise ValueError("Context must be a positive integer")
351 except (TypeError, ValueError):
350 except (TypeError, ValueError):
352 raise ValueError("Context must be a positive integer")
351 raise ValueError("Context must be a positive integer")
353 print(self.format_stack_entry(frame_lineno, '', context))
352 print(self.format_stack_entry(frame_lineno, '', context))
354
353
355 # vds: >>
354 # vds: >>
356 frame, lineno = frame_lineno
355 frame, lineno = frame_lineno
357 filename = frame.f_code.co_filename
356 filename = frame.f_code.co_filename
358 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
357 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
359 # vds: <<
358 # vds: <<
360
359
361 def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
360 def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
362 if context is None:
361 if context is None:
363 context = self.context
362 context = self.context
364 try:
363 try:
365 context=int(context)
364 context=int(context)
366 if context <= 0:
365 if context <= 0:
367 print("Context must be a positive integer")
366 print("Context must be a positive integer")
368 except (TypeError, ValueError):
367 except (TypeError, ValueError):
369 print("Context must be a positive integer")
368 print("Context must be a positive integer")
370 try:
369 try:
371 import reprlib # Py 3
370 import reprlib # Py 3
372 except ImportError:
371 except ImportError:
373 import repr as reprlib # Py 2
372 import repr as reprlib # Py 2
374
373
375 ret = []
374 ret = []
376
375
377 Colors = self.color_scheme_table.active_colors
376 Colors = self.color_scheme_table.active_colors
378 ColorsNormal = Colors.Normal
377 ColorsNormal = Colors.Normal
379 tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
378 tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
380 tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
379 tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
381 tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
380 tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
382 tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
381 tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
383 ColorsNormal)
382 ColorsNormal)
384
383
385 frame, lineno = frame_lineno
384 frame, lineno = frame_lineno
386
385
387 return_value = ''
386 return_value = ''
388 if '__return__' in frame.f_locals:
387 if '__return__' in frame.f_locals:
389 rv = frame.f_locals['__return__']
388 rv = frame.f_locals['__return__']
390 #return_value += '->'
389 #return_value += '->'
391 return_value += reprlib.repr(rv) + '\n'
390 return_value += reprlib.repr(rv) + '\n'
392 ret.append(return_value)
391 ret.append(return_value)
393
392
394 #s = filename + '(' + `lineno` + ')'
393 #s = filename + '(' + `lineno` + ')'
395 filename = self.canonic(frame.f_code.co_filename)
394 filename = self.canonic(frame.f_code.co_filename)
396 link = tpl_link % py3compat.cast_unicode(filename)
395 link = tpl_link % py3compat.cast_unicode(filename)
397
396
398 if frame.f_code.co_name:
397 if frame.f_code.co_name:
399 func = frame.f_code.co_name
398 func = frame.f_code.co_name
400 else:
399 else:
401 func = "<lambda>"
400 func = "<lambda>"
402
401
403 call = ''
402 call = ''
404 if func != '?':
403 if func != '?':
405 if '__args__' in frame.f_locals:
404 if '__args__' in frame.f_locals:
406 args = reprlib.repr(frame.f_locals['__args__'])
405 args = reprlib.repr(frame.f_locals['__args__'])
407 else:
406 else:
408 args = '()'
407 args = '()'
409 call = tpl_call % (func, args)
408 call = tpl_call % (func, args)
410
409
411 # The level info should be generated in the same format pdb uses, to
410 # The level info should be generated in the same format pdb uses, to
412 # avoid breaking the pdbtrack functionality of python-mode in *emacs.
411 # avoid breaking the pdbtrack functionality of python-mode in *emacs.
413 if frame is self.curframe:
412 if frame is self.curframe:
414 ret.append('> ')
413 ret.append('> ')
415 else:
414 else:
416 ret.append(' ')
415 ret.append(' ')
417 ret.append(u'%s(%s)%s\n' % (link,lineno,call))
416 ret.append(u'%s(%s)%s\n' % (link,lineno,call))
418
417
419 start = lineno - 1 - context//2
418 start = lineno - 1 - context//2
420 lines = linecache.getlines(filename)
419 lines = linecache.getlines(filename)
421 start = min(start, len(lines) - context)
420 start = min(start, len(lines) - context)
422 start = max(start, 0)
421 start = max(start, 0)
423 lines = lines[start : start + context]
422 lines = lines[start : start + context]
424
423
425 for i,line in enumerate(lines):
424 for i,line in enumerate(lines):
426 show_arrow = (start + 1 + i == lineno)
425 show_arrow = (start + 1 + i == lineno)
427 linetpl = (frame is self.curframe or show_arrow) \
426 linetpl = (frame is self.curframe or show_arrow) \
428 and tpl_line_em \
427 and tpl_line_em \
429 or tpl_line
428 or tpl_line
430 ret.append(self.__format_line(linetpl, filename,
429 ret.append(self.__format_line(linetpl, filename,
431 start + 1 + i, line,
430 start + 1 + i, line,
432 arrow = show_arrow) )
431 arrow = show_arrow) )
433 return ''.join(ret)
432 return ''.join(ret)
434
433
435 def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
434 def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
436 bp_mark = ""
435 bp_mark = ""
437 bp_mark_color = ""
436 bp_mark_color = ""
438
437
439 new_line, err = self.parser.format2(line, 'str')
438 new_line, err = self.parser.format2(line, 'str')
440 if not err:
439 if not err:
441 line = new_line
440 line = new_line
442
441
443 bp = None
442 bp = None
444 if lineno in self.get_file_breaks(filename):
443 if lineno in self.get_file_breaks(filename):
445 bps = self.get_breaks(filename, lineno)
444 bps = self.get_breaks(filename, lineno)
446 bp = bps[-1]
445 bp = bps[-1]
447
446
448 if bp:
447 if bp:
449 Colors = self.color_scheme_table.active_colors
448 Colors = self.color_scheme_table.active_colors
450 bp_mark = str(bp.number)
449 bp_mark = str(bp.number)
451 bp_mark_color = Colors.breakpoint_enabled
450 bp_mark_color = Colors.breakpoint_enabled
452 if not bp.enabled:
451 if not bp.enabled:
453 bp_mark_color = Colors.breakpoint_disabled
452 bp_mark_color = Colors.breakpoint_disabled
454
453
455 numbers_width = 7
454 numbers_width = 7
456 if arrow:
455 if arrow:
457 # This is the line with the error
456 # This is the line with the error
458 pad = numbers_width - len(str(lineno)) - len(bp_mark)
457 pad = numbers_width - len(str(lineno)) - len(bp_mark)
459 num = '%s%s' % (make_arrow(pad), str(lineno))
458 num = '%s%s' % (make_arrow(pad), str(lineno))
460 else:
459 else:
461 num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
460 num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
462
461
463 return tpl_line % (bp_mark_color + bp_mark, num, line)
462 return tpl_line % (bp_mark_color + bp_mark, num, line)
464
463
465
464
466 def print_list_lines(self, filename, first, last):
465 def print_list_lines(self, filename, first, last):
467 """The printing (as opposed to the parsing part of a 'list'
466 """The printing (as opposed to the parsing part of a 'list'
468 command."""
467 command."""
469 try:
468 try:
470 Colors = self.color_scheme_table.active_colors
469 Colors = self.color_scheme_table.active_colors
471 ColorsNormal = Colors.Normal
470 ColorsNormal = Colors.Normal
472 tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
471 tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
473 tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
472 tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
474 src = []
473 src = []
475 if filename == "<string>" and hasattr(self, "_exec_filename"):
474 if filename == "<string>" and hasattr(self, "_exec_filename"):
476 filename = self._exec_filename
475 filename = self._exec_filename
477
476
478 for lineno in range(first, last+1):
477 for lineno in range(first, last+1):
479 line = linecache.getline(filename, lineno)
478 line = linecache.getline(filename, lineno)
480 if not line:
479 if not line:
481 break
480 break
482
481
483 if lineno == self.curframe.f_lineno:
482 if lineno == self.curframe.f_lineno:
484 line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
483 line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
485 else:
484 else:
486 line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
485 line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
487
486
488 src.append(line)
487 src.append(line)
489 self.lineno = lineno
488 self.lineno = lineno
490
489
491 print(''.join(src))
490 print(''.join(src))
492
491
493 except KeyboardInterrupt:
492 except KeyboardInterrupt:
494 pass
493 pass
495
494
496 def do_list(self, arg):
495 def do_list(self, arg):
497 """Print lines of code from the current stack frame
496 """Print lines of code from the current stack frame
498 """
497 """
499 self.lastcmd = 'list'
498 self.lastcmd = 'list'
500 last = None
499 last = None
501 if arg:
500 if arg:
502 try:
501 try:
503 x = eval(arg, {}, {})
502 x = eval(arg, {}, {})
504 if type(x) == type(()):
503 if type(x) == type(()):
505 first, last = x
504 first, last = x
506 first = int(first)
505 first = int(first)
507 last = int(last)
506 last = int(last)
508 if last < first:
507 if last < first:
509 # Assume it's a count
508 # Assume it's a count
510 last = first + last
509 last = first + last
511 else:
510 else:
512 first = max(1, int(x) - 5)
511 first = max(1, int(x) - 5)
513 except:
512 except:
514 print('*** Error in argument:', repr(arg))
513 print('*** Error in argument:', repr(arg))
515 return
514 return
516 elif self.lineno is None:
515 elif self.lineno is None:
517 first = max(1, self.curframe.f_lineno - 5)
516 first = max(1, self.curframe.f_lineno - 5)
518 else:
517 else:
519 first = self.lineno + 1
518 first = self.lineno + 1
520 if last is None:
519 if last is None:
521 last = first + 10
520 last = first + 10
522 self.print_list_lines(self.curframe.f_code.co_filename, first, last)
521 self.print_list_lines(self.curframe.f_code.co_filename, first, last)
523
522
524 # vds: >>
523 # vds: >>
525 lineno = first
524 lineno = first
526 filename = self.curframe.f_code.co_filename
525 filename = self.curframe.f_code.co_filename
527 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
526 self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
528 # vds: <<
527 # vds: <<
529
528
530 do_l = do_list
529 do_l = do_list
531
530
532 def getsourcelines(self, obj):
531 def getsourcelines(self, obj):
533 lines, lineno = inspect.findsource(obj)
532 lines, lineno = inspect.findsource(obj)
534 if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
533 if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
535 # must be a module frame: do not try to cut a block out of it
534 # must be a module frame: do not try to cut a block out of it
536 return lines, 1
535 return lines, 1
537 elif inspect.ismodule(obj):
536 elif inspect.ismodule(obj):
538 return lines, 1
537 return lines, 1
539 return inspect.getblock(lines[lineno:]), lineno+1
538 return inspect.getblock(lines[lineno:]), lineno+1
540
539
541 def do_longlist(self, arg):
540 def do_longlist(self, arg):
542 """Print lines of code from the current stack frame.
541 """Print lines of code from the current stack frame.
543
542
544 Shows more lines than 'list' does.
543 Shows more lines than 'list' does.
545 """
544 """
546 self.lastcmd = 'longlist'
545 self.lastcmd = 'longlist'
547 try:
546 try:
548 lines, lineno = self.getsourcelines(self.curframe)
547 lines, lineno = self.getsourcelines(self.curframe)
549 except OSError as err:
548 except OSError as err:
550 self.error(err)
549 self.error(err)
551 return
550 return
552 last = lineno + len(lines)
551 last = lineno + len(lines)
553 self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
552 self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
554 do_ll = do_longlist
553 do_ll = do_longlist
555
554
556 def do_debug(self, arg):
555 def do_debug(self, arg):
557 """debug code
556 """debug code
558 Enter a recursive debugger that steps through the code
557 Enter a recursive debugger that steps through the code
559 argument (which is an arbitrary expression or statement to be
558 argument (which is an arbitrary expression or statement to be
560 executed in the current environment).
559 executed in the current environment).
561 """
560 """
562 sys.settrace(None)
561 sys.settrace(None)
563 globals = self.curframe.f_globals
562 globals = self.curframe.f_globals
564 locals = self.curframe_locals
563 locals = self.curframe_locals
565 p = self.__class__(completekey=self.completekey,
564 p = self.__class__(completekey=self.completekey,
566 stdin=self.stdin, stdout=self.stdout)
565 stdin=self.stdin, stdout=self.stdout)
567 p.use_rawinput = self.use_rawinput
566 p.use_rawinput = self.use_rawinput
568 p.prompt = "(%s) " % self.prompt.strip()
567 p.prompt = "(%s) " % self.prompt.strip()
569 self.message("ENTERING RECURSIVE DEBUGGER")
568 self.message("ENTERING RECURSIVE DEBUGGER")
570 sys.call_tracing(p.run, (arg, globals, locals))
569 sys.call_tracing(p.run, (arg, globals, locals))
571 self.message("LEAVING RECURSIVE DEBUGGER")
570 self.message("LEAVING RECURSIVE DEBUGGER")
572 sys.settrace(self.trace_dispatch)
571 sys.settrace(self.trace_dispatch)
573 self.lastcmd = p.lastcmd
572 self.lastcmd = p.lastcmd
574
573
575 def do_pdef(self, arg):
574 def do_pdef(self, arg):
576 """Print the call signature for any callable object.
575 """Print the call signature for any callable object.
577
576
578 The debugger interface to %pdef"""
577 The debugger interface to %pdef"""
579 namespaces = [('Locals', self.curframe.f_locals),
578 namespaces = [('Locals', self.curframe.f_locals),
580 ('Globals', self.curframe.f_globals)]
579 ('Globals', self.curframe.f_globals)]
581 self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
580 self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
582
581
583 def do_pdoc(self, arg):
582 def do_pdoc(self, arg):
584 """Print the docstring for an object.
583 """Print the docstring for an object.
585
584
586 The debugger interface to %pdoc."""
585 The debugger interface to %pdoc."""
587 namespaces = [('Locals', self.curframe.f_locals),
586 namespaces = [('Locals', self.curframe.f_locals),
588 ('Globals', self.curframe.f_globals)]
587 ('Globals', self.curframe.f_globals)]
589 self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
588 self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
590
589
591 def do_pfile(self, arg):
590 def do_pfile(self, arg):
592 """Print (or run through pager) the file where an object is defined.
591 """Print (or run through pager) the file where an object is defined.
593
592
594 The debugger interface to %pfile.
593 The debugger interface to %pfile.
595 """
594 """
596 namespaces = [('Locals', self.curframe.f_locals),
595 namespaces = [('Locals', self.curframe.f_locals),
597 ('Globals', self.curframe.f_globals)]
596 ('Globals', self.curframe.f_globals)]
598 self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
597 self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
599
598
600 def do_pinfo(self, arg):
599 def do_pinfo(self, arg):
601 """Provide detailed information about an object.
600 """Provide detailed information about an object.
602
601
603 The debugger interface to %pinfo, i.e., obj?."""
602 The debugger interface to %pinfo, i.e., obj?."""
604 namespaces = [('Locals', self.curframe.f_locals),
603 namespaces = [('Locals', self.curframe.f_locals),
605 ('Globals', self.curframe.f_globals)]
604 ('Globals', self.curframe.f_globals)]
606 self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
605 self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
607
606
608 def do_pinfo2(self, arg):
607 def do_pinfo2(self, arg):
609 """Provide extra detailed information about an object.
608 """Provide extra detailed information about an object.
610
609
611 The debugger interface to %pinfo2, i.e., obj??."""
610 The debugger interface to %pinfo2, i.e., obj??."""
612 namespaces = [('Locals', self.curframe.f_locals),
611 namespaces = [('Locals', self.curframe.f_locals),
613 ('Globals', self.curframe.f_globals)]
612 ('Globals', self.curframe.f_globals)]
614 self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
613 self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
615
614
616 def do_psource(self, arg):
615 def do_psource(self, arg):
617 """Print (or run through pager) the source code for an object."""
616 """Print (or run through pager) the source code for an object."""
618 namespaces = [('Locals', self.curframe.f_locals),
617 namespaces = [('Locals', self.curframe.f_locals),
619 ('Globals', self.curframe.f_globals)]
618 ('Globals', self.curframe.f_globals)]
620 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
619 self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
621
620
622 def do_where(self, arg):
621 def do_where(self, arg):
623 """w(here)
622 """w(here)
624 Print a stack trace, with the most recent frame at the bottom.
623 Print a stack trace, with the most recent frame at the bottom.
625 An arrow indicates the "current frame", which determines the
624 An arrow indicates the "current frame", which determines the
626 context of most commands. 'bt' is an alias for this command.
625 context of most commands. 'bt' is an alias for this command.
627
626
628 Take a number as argument as an (optional) number of context line to
627 Take a number as argument as an (optional) number of context line to
629 print"""
628 print"""
630 if arg:
629 if arg:
631 context = int(arg)
630 context = int(arg)
632 self.print_stack_trace(context)
631 self.print_stack_trace(context)
633 else:
632 else:
634 self.print_stack_trace()
633 self.print_stack_trace()
635
634
636 do_w = do_where
635 do_w = do_where
637
636
638
637
639 def set_trace(frame=None):
638 def set_trace(frame=None):
640 """
639 """
641 Start debugging from `frame`.
640 Start debugging from `frame`.
642
641
643 If frame is not specified, debugging starts from caller's frame.
642 If frame is not specified, debugging starts from caller's frame.
644 """
643 """
645 Pdb().set_trace(frame or sys._getframe().f_back)
644 Pdb().set_trace(frame or sys._getframe().f_back)
@@ -1,3659 +1,3658 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Main IPython class."""
2 """Main IPython class."""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
5 # Copyright (C) 2001 Janko Hauser <jhauser@zscout.de>
6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
6 # Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
7 # Copyright (C) 2008-2011 The IPython Development Team
7 # Copyright (C) 2008-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13
13
14 import abc
14 import abc
15 import ast
15 import ast
16 import asyncio
16 import asyncio
17 import atexit
17 import atexit
18 import builtins as builtin_mod
18 import builtins as builtin_mod
19 import functools
19 import functools
20 import os
20 import os
21 import re
21 import re
22 import runpy
22 import runpy
23 import sys
23 import sys
24 import tempfile
24 import tempfile
25 import traceback
25 import traceback
26 import types
26 import types
27 import subprocess
27 import subprocess
28 import warnings
28 import warnings
29 from io import open as io_open
29 from io import open as io_open
30
30
31 from pickleshare import PickleShareDB
31 from pickleshare import PickleShareDB
32
32
33 from traitlets.config.configurable import SingletonConfigurable
33 from traitlets.config.configurable import SingletonConfigurable
34 from traitlets.utils.importstring import import_item
34 from traitlets.utils.importstring import import_item
35 from IPython.core import oinspect
35 from IPython.core import oinspect
36 from IPython.core import magic
36 from IPython.core import magic
37 from IPython.core import page
37 from IPython.core import page
38 from IPython.core import prefilter
38 from IPython.core import prefilter
39 from IPython.core import ultratb
39 from IPython.core import ultratb
40 from IPython.core.alias import Alias, AliasManager
40 from IPython.core.alias import Alias, AliasManager
41 from IPython.core.autocall import ExitAutocall
41 from IPython.core.autocall import ExitAutocall
42 from IPython.core.builtin_trap import BuiltinTrap
42 from IPython.core.builtin_trap import BuiltinTrap
43 from IPython.core.events import EventManager, available_events
43 from IPython.core.events import EventManager, available_events
44 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
44 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
45 from IPython.core.debugger import Pdb
45 from IPython.core.debugger import Pdb
46 from IPython.core.display_trap import DisplayTrap
46 from IPython.core.display_trap import DisplayTrap
47 from IPython.core.displayhook import DisplayHook
47 from IPython.core.displayhook import DisplayHook
48 from IPython.core.displaypub import DisplayPublisher
48 from IPython.core.displaypub import DisplayPublisher
49 from IPython.core.error import InputRejected, UsageError
49 from IPython.core.error import InputRejected, UsageError
50 from IPython.core.extensions import ExtensionManager
50 from IPython.core.extensions import ExtensionManager
51 from IPython.core.formatters import DisplayFormatter
51 from IPython.core.formatters import DisplayFormatter
52 from IPython.core.history import HistoryManager
52 from IPython.core.history import HistoryManager
53 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
53 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
54 from IPython.core.logger import Logger
54 from IPython.core.logger import Logger
55 from IPython.core.macro import Macro
55 from IPython.core.macro import Macro
56 from IPython.core.payload import PayloadManager
56 from IPython.core.payload import PayloadManager
57 from IPython.core.prefilter import PrefilterManager
57 from IPython.core.prefilter import PrefilterManager
58 from IPython.core.profiledir import ProfileDir
58 from IPython.core.profiledir import ProfileDir
59 from IPython.core.usage import default_banner
59 from IPython.core.usage import default_banner
60 from IPython.display import display
60 from IPython.display import display
61 from IPython.testing.skipdoctest import skip_doctest
61 from IPython.testing.skipdoctest import skip_doctest
62 from IPython.utils import PyColorize
62 from IPython.utils import PyColorize
63 from IPython.utils import io
63 from IPython.utils import io
64 from IPython.utils import py3compat
64 from IPython.utils import py3compat
65 from IPython.utils import openpy
65 from IPython.utils import openpy
66 from IPython.utils.decorators import undoc
66 from IPython.utils.decorators import undoc
67 from IPython.utils.io import ask_yes_no
67 from IPython.utils.io import ask_yes_no
68 from IPython.utils.ipstruct import Struct
68 from IPython.utils.ipstruct import Struct
69 from IPython.paths import get_ipython_dir
69 from IPython.paths import get_ipython_dir
70 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
70 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
71 from IPython.utils.process import system, getoutput
71 from IPython.utils.process import system, getoutput
72 from IPython.utils.strdispatch import StrDispatch
72 from IPython.utils.strdispatch import StrDispatch
73 from IPython.utils.syspathcontext import prepended_to_syspath
73 from IPython.utils.syspathcontext import prepended_to_syspath
74 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
74 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
75 from IPython.utils.tempdir import TemporaryDirectory
75 from IPython.utils.tempdir import TemporaryDirectory
76 from traitlets import (
76 from traitlets import (
77 Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
77 Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
78 observe, default, validate, Any
78 observe, default, validate, Any
79 )
79 )
80 from warnings import warn
80 from warnings import warn
81 from logging import error
81 from logging import error
82 import IPython.core.hooks
82 import IPython.core.hooks
83
83
84 from typing import List as ListType, Tuple
84 from typing import List as ListType, Tuple
85 from ast import AST
85 from ast import AST
86
86
87 # NoOpContext is deprecated, but ipykernel imports it from here.
87 # NoOpContext is deprecated, but ipykernel imports it from here.
88 # See https://github.com/ipython/ipykernel/issues/157
88 # See https://github.com/ipython/ipykernel/issues/157
89 from IPython.utils.contexts import NoOpContext
89 from IPython.utils.contexts import NoOpContext
90
90
91 try:
91 try:
92 import docrepr.sphinxify as sphx
92 import docrepr.sphinxify as sphx
93
93
94 def sphinxify(doc):
94 def sphinxify(doc):
95 with TemporaryDirectory() as dirname:
95 with TemporaryDirectory() as dirname:
96 return {
96 return {
97 'text/html': sphx.sphinxify(doc, dirname),
97 'text/html': sphx.sphinxify(doc, dirname),
98 'text/plain': doc
98 'text/plain': doc
99 }
99 }
100 except ImportError:
100 except ImportError:
101 sphinxify = None
101 sphinxify = None
102
102
103
103
104 class ProvisionalWarning(DeprecationWarning):
104 class ProvisionalWarning(DeprecationWarning):
105 """
105 """
106 Warning class for unstable features
106 Warning class for unstable features
107 """
107 """
108 pass
108 pass
109
109
110 if sys.version_info > (3,6):
110 if sys.version_info > (3,6):
111 _assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign)
111 _assign_nodes = (ast.AugAssign, ast.AnnAssign, ast.Assign)
112 _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
112 _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
113 else:
113 else:
114 _assign_nodes = (ast.AugAssign, ast.Assign )
114 _assign_nodes = (ast.AugAssign, ast.Assign )
115 _single_targets_nodes = (ast.AugAssign, )
115 _single_targets_nodes = (ast.AugAssign, )
116
116
117 #-----------------------------------------------------------------------------
117 #-----------------------------------------------------------------------------
118 # Await Helpers
118 # Await Helpers
119 #-----------------------------------------------------------------------------
119 #-----------------------------------------------------------------------------
120
120
121 def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
121 def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
122 """Return a function that do not create a new local scope.
122 """Return a function that do not create a new local scope.
123
123
124 Given a function, create a clone of this function where the co_newlocal flag
124 Given a function, create a clone of this function where the co_newlocal flag
125 has been removed, making this function code actually run in the sourounding
125 has been removed, making this function code actually run in the sourounding
126 scope.
126 scope.
127
127
128 We need this in order to run asynchronous code in user level namespace.
128 We need this in order to run asynchronous code in user level namespace.
129 """
129 """
130 from types import CodeType, FunctionType
130 from types import CodeType, FunctionType
131 CO_NEWLOCALS = 0x0002
131 CO_NEWLOCALS = 0x0002
132 code = function.__code__
132 code = function.__code__
133 new_code = CodeType(
133 new_code = CodeType(
134 code.co_argcount,
134 code.co_argcount,
135 code.co_kwonlyargcount,
135 code.co_kwonlyargcount,
136 code.co_nlocals,
136 code.co_nlocals,
137 code.co_stacksize,
137 code.co_stacksize,
138 code.co_flags & ~CO_NEWLOCALS,
138 code.co_flags & ~CO_NEWLOCALS,
139 code.co_code,
139 code.co_code,
140 code.co_consts,
140 code.co_consts,
141 code.co_names,
141 code.co_names,
142 code.co_varnames,
142 code.co_varnames,
143 code.co_filename,
143 code.co_filename,
144 code.co_name,
144 code.co_name,
145 code.co_firstlineno,
145 code.co_firstlineno,
146 code.co_lnotab,
146 code.co_lnotab,
147 code.co_freevars,
147 code.co_freevars,
148 code.co_cellvars
148 code.co_cellvars
149 )
149 )
150 return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
150 return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
151
151
152
152
153 # we still need to run things using the asyncio eventloop, but there is no
153 # we still need to run things using the asyncio eventloop, but there is no
154 # async integration
154 # async integration
155 from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner)
155 from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner)
156
156
157 if sys.version_info > (3, 5):
157 if sys.version_info > (3, 5):
158 from .async_helpers import _curio_runner, _trio_runner, _should_be_async
158 from .async_helpers import _curio_runner, _trio_runner, _should_be_async
159 else :
159 else :
160 _curio_runner = _trio_runner = None
160 _curio_runner = _trio_runner = None
161
161
162 def _should_be_async(cell:str)->bool:
162 def _should_be_async(cell:str)->bool:
163 return False
163 return False
164
164
165
165
166 def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
166 def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
167 """
167 """
168 Parse a cell with top-level await and modify the AST to be able to run it later.
168 Parse a cell with top-level await and modify the AST to be able to run it later.
169
169
170 Parameter
170 Parameter
171 ---------
171 ---------
172
172
173 cell: str
173 cell: str
174 The code cell to asyncronify
174 The code cell to asyncronify
175 wrapper_name: str
175 wrapper_name: str
176 The name of the function to be used to wrap the passed `cell`. It is
176 The name of the function to be used to wrap the passed `cell`. It is
177 advised to **not** use a python identifier in order to not pollute the
177 advised to **not** use a python identifier in order to not pollute the
178 global namespace in which the function will be ran.
178 global namespace in which the function will be ran.
179
179
180 Return
180 Return
181 ------
181 ------
182
182
183 A module object AST containing **one** function named `wrapper_name`.
183 A module object AST containing **one** function named `wrapper_name`.
184
184
185 The given code is wrapped in a async-def function, parsed into an AST, and
185 The given code is wrapped in a async-def function, parsed into an AST, and
186 the resulting function definition AST is modified to return the last
186 the resulting function definition AST is modified to return the last
187 expression.
187 expression.
188
188
189 The last expression or await node is moved into a return statement at the
189 The last expression or await node is moved into a return statement at the
190 end of the function, and removed from its original location. If the last
190 end of the function, and removed from its original location. If the last
191 node is not Expr or Await nothing is done.
191 node is not Expr or Await nothing is done.
192
192
193 The function `__code__` will need to be later modified (by
193 The function `__code__` will need to be later modified (by
194 ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
194 ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
195 meaning that the local and global scope are the same, ie as if the body of
195 meaning that the local and global scope are the same, ie as if the body of
196 the function was at module level.
196 the function was at module level.
197
197
198 Lastly a call to `locals()` is made just before the last expression of the
198 Lastly a call to `locals()` is made just before the last expression of the
199 function, or just after the last assignment or statement to make sure the
199 function, or just after the last assignment or statement to make sure the
200 global dict is updated as python function work with a local fast cache which
200 global dict is updated as python function work with a local fast cache which
201 is updated only on `local()` calls.
201 is updated only on `local()` calls.
202 """
202 """
203
203
204 from ast import Expr, Await, Return
204 from ast import Expr, Await, Return
205 tree = ast.parse(_asyncify(cell))
205 tree = ast.parse(_asyncify(cell))
206
206
207 function_def = tree.body[0]
207 function_def = tree.body[0]
208 function_def.name = wrapper_name
208 function_def.name = wrapper_name
209 try_block = function_def.body[0]
209 try_block = function_def.body[0]
210 lastexpr = try_block.body[-1]
210 lastexpr = try_block.body[-1]
211 if isinstance(lastexpr, (Expr, Await)):
211 if isinstance(lastexpr, (Expr, Await)):
212 try_block.body[-1] = Return(lastexpr.value)
212 try_block.body[-1] = Return(lastexpr.value)
213 ast.fix_missing_locations(tree)
213 ast.fix_missing_locations(tree)
214 return tree
214 return tree
215 #-----------------------------------------------------------------------------
215 #-----------------------------------------------------------------------------
216 # Globals
216 # Globals
217 #-----------------------------------------------------------------------------
217 #-----------------------------------------------------------------------------
218
218
219 # compiled regexps for autoindent management
219 # compiled regexps for autoindent management
220 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
220 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
221
221
222 #-----------------------------------------------------------------------------
222 #-----------------------------------------------------------------------------
223 # Utilities
223 # Utilities
224 #-----------------------------------------------------------------------------
224 #-----------------------------------------------------------------------------
225
225
226 @undoc
226 @undoc
227 def softspace(file, newvalue):
227 def softspace(file, newvalue):
228 """Copied from code.py, to remove the dependency"""
228 """Copied from code.py, to remove the dependency"""
229
229
230 oldvalue = 0
230 oldvalue = 0
231 try:
231 try:
232 oldvalue = file.softspace
232 oldvalue = file.softspace
233 except AttributeError:
233 except AttributeError:
234 pass
234 pass
235 try:
235 try:
236 file.softspace = newvalue
236 file.softspace = newvalue
237 except (AttributeError, TypeError):
237 except (AttributeError, TypeError):
238 # "attribute-less object" or "read-only attributes"
238 # "attribute-less object" or "read-only attributes"
239 pass
239 pass
240 return oldvalue
240 return oldvalue
241
241
242 @undoc
242 @undoc
243 def no_op(*a, **kw):
243 def no_op(*a, **kw):
244 pass
244 pass
245
245
246
246
247 class SpaceInInput(Exception): pass
247 class SpaceInInput(Exception): pass
248
248
249
249
250 def get_default_colors():
250 def get_default_colors():
251 "DEPRECATED"
251 "DEPRECATED"
252 warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.',
252 warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.',
253 DeprecationWarning, stacklevel=2)
253 DeprecationWarning, stacklevel=2)
254 return 'Neutral'
254 return 'Neutral'
255
255
256
256
257 class SeparateUnicode(Unicode):
257 class SeparateUnicode(Unicode):
258 r"""A Unicode subclass to validate separate_in, separate_out, etc.
258 r"""A Unicode subclass to validate separate_in, separate_out, etc.
259
259
260 This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
260 This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
261 """
261 """
262
262
263 def validate(self, obj, value):
263 def validate(self, obj, value):
264 if value == '0': value = ''
264 if value == '0': value = ''
265 value = value.replace('\\n','\n')
265 value = value.replace('\\n','\n')
266 return super(SeparateUnicode, self).validate(obj, value)
266 return super(SeparateUnicode, self).validate(obj, value)
267
267
268
268
269 @undoc
269 @undoc
270 class DummyMod(object):
270 class DummyMod(object):
271 """A dummy module used for IPython's interactive module when
271 """A dummy module used for IPython's interactive module when
272 a namespace must be assigned to the module's __dict__."""
272 a namespace must be assigned to the module's __dict__."""
273 __spec__ = None
273 __spec__ = None
274
274
275
275
276 class ExecutionInfo(object):
276 class ExecutionInfo(object):
277 """The arguments used for a call to :meth:`InteractiveShell.run_cell`
277 """The arguments used for a call to :meth:`InteractiveShell.run_cell`
278
278
279 Stores information about what is going to happen.
279 Stores information about what is going to happen.
280 """
280 """
281 raw_cell = None
281 raw_cell = None
282 store_history = False
282 store_history = False
283 silent = False
283 silent = False
284 shell_futures = True
284 shell_futures = True
285
285
286 def __init__(self, raw_cell, store_history, silent, shell_futures):
286 def __init__(self, raw_cell, store_history, silent, shell_futures):
287 self.raw_cell = raw_cell
287 self.raw_cell = raw_cell
288 self.store_history = store_history
288 self.store_history = store_history
289 self.silent = silent
289 self.silent = silent
290 self.shell_futures = shell_futures
290 self.shell_futures = shell_futures
291
291
292 def __repr__(self):
292 def __repr__(self):
293 name = self.__class__.__qualname__
293 name = self.__class__.__qualname__
294 raw_cell = ((self.raw_cell[:50] + '..')
294 raw_cell = ((self.raw_cell[:50] + '..')
295 if len(self.raw_cell) > 50 else self.raw_cell)
295 if len(self.raw_cell) > 50 else self.raw_cell)
296 return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s>' %\
296 return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s>' %\
297 (name, id(self), raw_cell, self.store_history, self.silent, self.shell_futures)
297 (name, id(self), raw_cell, self.store_history, self.silent, self.shell_futures)
298
298
299
299
300 class ExecutionResult(object):
300 class ExecutionResult(object):
301 """The result of a call to :meth:`InteractiveShell.run_cell`
301 """The result of a call to :meth:`InteractiveShell.run_cell`
302
302
303 Stores information about what took place.
303 Stores information about what took place.
304 """
304 """
305 execution_count = None
305 execution_count = None
306 error_before_exec = None
306 error_before_exec = None
307 error_in_exec = None
307 error_in_exec = None
308 info = None
308 info = None
309 result = None
309 result = None
310
310
311 def __init__(self, info):
311 def __init__(self, info):
312 self.info = info
312 self.info = info
313
313
314 @property
314 @property
315 def success(self):
315 def success(self):
316 return (self.error_before_exec is None) and (self.error_in_exec is None)
316 return (self.error_before_exec is None) and (self.error_in_exec is None)
317
317
318 def raise_error(self):
318 def raise_error(self):
319 """Reraises error if `success` is `False`, otherwise does nothing"""
319 """Reraises error if `success` is `False`, otherwise does nothing"""
320 if self.error_before_exec is not None:
320 if self.error_before_exec is not None:
321 raise self.error_before_exec
321 raise self.error_before_exec
322 if self.error_in_exec is not None:
322 if self.error_in_exec is not None:
323 raise self.error_in_exec
323 raise self.error_in_exec
324
324
325 def __repr__(self):
325 def __repr__(self):
326 name = self.__class__.__qualname__
326 name = self.__class__.__qualname__
327 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
327 return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
328 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
328 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
329
329
330
330
331 class InteractiveShell(SingletonConfigurable):
331 class InteractiveShell(SingletonConfigurable):
332 """An enhanced, interactive shell for Python."""
332 """An enhanced, interactive shell for Python."""
333
333
334 _instance = None
334 _instance = None
335
335
336 ast_transformers = List([], help=
336 ast_transformers = List([], help=
337 """
337 """
338 A list of ast.NodeTransformer subclass instances, which will be applied
338 A list of ast.NodeTransformer subclass instances, which will be applied
339 to user input before code is run.
339 to user input before code is run.
340 """
340 """
341 ).tag(config=True)
341 ).tag(config=True)
342
342
343 autocall = Enum((0,1,2), default_value=0, help=
343 autocall = Enum((0,1,2), default_value=0, help=
344 """
344 """
345 Make IPython automatically call any callable object even if you didn't
345 Make IPython automatically call any callable object even if you didn't
346 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
346 type explicit parentheses. For example, 'str 43' becomes 'str(43)'
347 automatically. The value can be '0' to disable the feature, '1' for
347 automatically. The value can be '0' to disable the feature, '1' for
348 'smart' autocall, where it is not applied if there are no more
348 'smart' autocall, where it is not applied if there are no more
349 arguments on the line, and '2' for 'full' autocall, where all callable
349 arguments on the line, and '2' for 'full' autocall, where all callable
350 objects are automatically called (even if no arguments are present).
350 objects are automatically called (even if no arguments are present).
351 """
351 """
352 ).tag(config=True)
352 ).tag(config=True)
353
353
354 autoindent = Bool(True, help=
354 autoindent = Bool(True, help=
355 """
355 """
356 Autoindent IPython code entered interactively.
356 Autoindent IPython code entered interactively.
357 """
357 """
358 ).tag(config=True)
358 ).tag(config=True)
359
359
360 autoawait = Bool(True, help=
360 autoawait = Bool(True, help=
361 """
361 """
362 Automatically run await statement in the top level repl.
362 Automatically run await statement in the top level repl.
363 """
363 """
364 ).tag(config=True)
364 ).tag(config=True)
365
365
366 loop_runner_map ={
366 loop_runner_map ={
367 'asyncio':(_asyncio_runner, True),
367 'asyncio':(_asyncio_runner, True),
368 'curio':(_curio_runner, True),
368 'curio':(_curio_runner, True),
369 'trio':(_trio_runner, True),
369 'trio':(_trio_runner, True),
370 'sync': (_pseudo_sync_runner, False)
370 'sync': (_pseudo_sync_runner, False)
371 }
371 }
372
372
373 loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
373 loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
374 allow_none=True,
374 allow_none=True,
375 help="""Select the loop runner that will be used to execute top-level asynchronous code"""
375 help="""Select the loop runner that will be used to execute top-level asynchronous code"""
376 ).tag(config=True)
376 ).tag(config=True)
377
377
378 @default('loop_runner')
378 @default('loop_runner')
379 def _default_loop_runner(self):
379 def _default_loop_runner(self):
380 return import_item("IPython.core.interactiveshell._asyncio_runner")
380 return import_item("IPython.core.interactiveshell._asyncio_runner")
381
381
382 @validate('loop_runner')
382 @validate('loop_runner')
383 def _import_runner(self, proposal):
383 def _import_runner(self, proposal):
384 if isinstance(proposal.value, str):
384 if isinstance(proposal.value, str):
385 if proposal.value in self.loop_runner_map:
385 if proposal.value in self.loop_runner_map:
386 runner, autoawait = self.loop_runner_map[proposal.value]
386 runner, autoawait = self.loop_runner_map[proposal.value]
387 self.autoawait = autoawait
387 self.autoawait = autoawait
388 return runner
388 return runner
389 runner = import_item(proposal.value)
389 runner = import_item(proposal.value)
390 if not callable(runner):
390 if not callable(runner):
391 raise ValueError('loop_runner must be callable')
391 raise ValueError('loop_runner must be callable')
392 return runner
392 return runner
393 if not callable(proposal.value):
393 if not callable(proposal.value):
394 raise ValueError('loop_runner must be callable')
394 raise ValueError('loop_runner must be callable')
395 return proposal.value
395 return proposal.value
396
396
397 automagic = Bool(True, help=
397 automagic = Bool(True, help=
398 """
398 """
399 Enable magic commands to be called without the leading %.
399 Enable magic commands to be called without the leading %.
400 """
400 """
401 ).tag(config=True)
401 ).tag(config=True)
402
402
403 banner1 = Unicode(default_banner,
403 banner1 = Unicode(default_banner,
404 help="""The part of the banner to be printed before the profile"""
404 help="""The part of the banner to be printed before the profile"""
405 ).tag(config=True)
405 ).tag(config=True)
406 banner2 = Unicode('',
406 banner2 = Unicode('',
407 help="""The part of the banner to be printed after the profile"""
407 help="""The part of the banner to be printed after the profile"""
408 ).tag(config=True)
408 ).tag(config=True)
409
409
410 cache_size = Integer(1000, help=
410 cache_size = Integer(1000, help=
411 """
411 """
412 Set the size of the output cache. The default is 1000, you can
412 Set the size of the output cache. The default is 1000, you can
413 change it permanently in your config file. Setting it to 0 completely
413 change it permanently in your config file. Setting it to 0 completely
414 disables the caching system, and the minimum value accepted is 3 (if
414 disables the caching system, and the minimum value accepted is 3 (if
415 you provide a value less than 3, it is reset to 0 and a warning is
415 you provide a value less than 3, it is reset to 0 and a warning is
416 issued). This limit is defined because otherwise you'll spend more
416 issued). This limit is defined because otherwise you'll spend more
417 time re-flushing a too small cache than working
417 time re-flushing a too small cache than working
418 """
418 """
419 ).tag(config=True)
419 ).tag(config=True)
420 color_info = Bool(True, help=
420 color_info = Bool(True, help=
421 """
421 """
422 Use colors for displaying information about objects. Because this
422 Use colors for displaying information about objects. Because this
423 information is passed through a pager (like 'less'), and some pagers
423 information is passed through a pager (like 'less'), and some pagers
424 get confused with color codes, this capability can be turned off.
424 get confused with color codes, this capability can be turned off.
425 """
425 """
426 ).tag(config=True)
426 ).tag(config=True)
427 colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
427 colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
428 default_value='Neutral',
428 default_value='Neutral',
429 help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
429 help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
430 ).tag(config=True)
430 ).tag(config=True)
431 debug = Bool(False).tag(config=True)
431 debug = Bool(False).tag(config=True)
432 disable_failing_post_execute = Bool(False,
432 disable_failing_post_execute = Bool(False,
433 help="Don't call post-execute functions that have failed in the past."
433 help="Don't call post-execute functions that have failed in the past."
434 ).tag(config=True)
434 ).tag(config=True)
435 display_formatter = Instance(DisplayFormatter, allow_none=True)
435 display_formatter = Instance(DisplayFormatter, allow_none=True)
436 displayhook_class = Type(DisplayHook)
436 displayhook_class = Type(DisplayHook)
437 display_pub_class = Type(DisplayPublisher)
437 display_pub_class = Type(DisplayPublisher)
438
438
439 sphinxify_docstring = Bool(False, help=
439 sphinxify_docstring = Bool(False, help=
440 """
440 """
441 Enables rich html representation of docstrings. (This requires the
441 Enables rich html representation of docstrings. (This requires the
442 docrepr module).
442 docrepr module).
443 """).tag(config=True)
443 """).tag(config=True)
444
444
445 @observe("sphinxify_docstring")
445 @observe("sphinxify_docstring")
446 def _sphinxify_docstring_changed(self, change):
446 def _sphinxify_docstring_changed(self, change):
447 if change['new']:
447 if change['new']:
448 warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
448 warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
449
449
450 enable_html_pager = Bool(False, help=
450 enable_html_pager = Bool(False, help=
451 """
451 """
452 (Provisional API) enables html representation in mime bundles sent
452 (Provisional API) enables html representation in mime bundles sent
453 to pagers.
453 to pagers.
454 """).tag(config=True)
454 """).tag(config=True)
455
455
456 @observe("enable_html_pager")
456 @observe("enable_html_pager")
457 def _enable_html_pager_changed(self, change):
457 def _enable_html_pager_changed(self, change):
458 if change['new']:
458 if change['new']:
459 warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
459 warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
460
460
461 data_pub_class = None
461 data_pub_class = None
462
462
463 exit_now = Bool(False)
463 exit_now = Bool(False)
464 exiter = Instance(ExitAutocall)
464 exiter = Instance(ExitAutocall)
465 @default('exiter')
465 @default('exiter')
466 def _exiter_default(self):
466 def _exiter_default(self):
467 return ExitAutocall(self)
467 return ExitAutocall(self)
468 # Monotonically increasing execution counter
468 # Monotonically increasing execution counter
469 execution_count = Integer(1)
469 execution_count = Integer(1)
470 filename = Unicode("<ipython console>")
470 filename = Unicode("<ipython console>")
471 ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
471 ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
472
472
473 # Used to transform cells before running them, and check whether code is complete
473 # Used to transform cells before running them, and check whether code is complete
474 input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
474 input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
475 ())
475 ())
476
476
477 @property
477 @property
478 def input_transformers_cleanup(self):
478 def input_transformers_cleanup(self):
479 return self.input_transformer_manager.cleanup_transforms
479 return self.input_transformer_manager.cleanup_transforms
480
480
481 input_transformers_post = List([],
481 input_transformers_post = List([],
482 help="A list of string input transformers, to be applied after IPython's "
482 help="A list of string input transformers, to be applied after IPython's "
483 "own input transformations."
483 "own input transformations."
484 )
484 )
485
485
486 @property
486 @property
487 def input_splitter(self):
487 def input_splitter(self):
488 """Make this available for backward compatibility (pre-7.0 release) with existing code.
488 """Make this available for backward compatibility (pre-7.0 release) with existing code.
489
489
490 For example, ipykernel ipykernel currently uses
490 For example, ipykernel ipykernel currently uses
491 `shell.input_splitter.check_complete`
491 `shell.input_splitter.check_complete`
492 """
492 """
493 from warnings import warn
493 from warnings import warn
494 warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
494 warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
495 DeprecationWarning, stacklevel=2
495 DeprecationWarning, stacklevel=2
496 )
496 )
497 return self.input_transformer_manager
497 return self.input_transformer_manager
498
498
499 logstart = Bool(False, help=
499 logstart = Bool(False, help=
500 """
500 """
501 Start logging to the default log file in overwrite mode.
501 Start logging to the default log file in overwrite mode.
502 Use `logappend` to specify a log file to **append** logs to.
502 Use `logappend` to specify a log file to **append** logs to.
503 """
503 """
504 ).tag(config=True)
504 ).tag(config=True)
505 logfile = Unicode('', help=
505 logfile = Unicode('', help=
506 """
506 """
507 The name of the logfile to use.
507 The name of the logfile to use.
508 """
508 """
509 ).tag(config=True)
509 ).tag(config=True)
510 logappend = Unicode('', help=
510 logappend = Unicode('', help=
511 """
511 """
512 Start logging to the given file in append mode.
512 Start logging to the given file in append mode.
513 Use `logfile` to specify a log file to **overwrite** logs to.
513 Use `logfile` to specify a log file to **overwrite** logs to.
514 """
514 """
515 ).tag(config=True)
515 ).tag(config=True)
516 object_info_string_level = Enum((0,1,2), default_value=0,
516 object_info_string_level = Enum((0,1,2), default_value=0,
517 ).tag(config=True)
517 ).tag(config=True)
518 pdb = Bool(False, help=
518 pdb = Bool(False, help=
519 """
519 """
520 Automatically call the pdb debugger after every exception.
520 Automatically call the pdb debugger after every exception.
521 """
521 """
522 ).tag(config=True)
522 ).tag(config=True)
523 display_page = Bool(False,
523 display_page = Bool(False,
524 help="""If True, anything that would be passed to the pager
524 help="""If True, anything that would be passed to the pager
525 will be displayed as regular output instead."""
525 will be displayed as regular output instead."""
526 ).tag(config=True)
526 ).tag(config=True)
527
527
528 # deprecated prompt traits:
528 # deprecated prompt traits:
529
529
530 prompt_in1 = Unicode('In [\\#]: ',
530 prompt_in1 = Unicode('In [\\#]: ',
531 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
531 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
532 ).tag(config=True)
532 ).tag(config=True)
533 prompt_in2 = Unicode(' .\\D.: ',
533 prompt_in2 = Unicode(' .\\D.: ',
534 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
534 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
535 ).tag(config=True)
535 ).tag(config=True)
536 prompt_out = Unicode('Out[\\#]: ',
536 prompt_out = Unicode('Out[\\#]: ',
537 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
537 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
538 ).tag(config=True)
538 ).tag(config=True)
539 prompts_pad_left = Bool(True,
539 prompts_pad_left = Bool(True,
540 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
540 help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
541 ).tag(config=True)
541 ).tag(config=True)
542
542
543 @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
543 @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
544 def _prompt_trait_changed(self, change):
544 def _prompt_trait_changed(self, change):
545 name = change['name']
545 name = change['name']
546 warn("InteractiveShell.{name} is deprecated since IPython 4.0"
546 warn("InteractiveShell.{name} is deprecated since IPython 4.0"
547 " and ignored since 5.0, set TerminalInteractiveShell.prompts"
547 " and ignored since 5.0, set TerminalInteractiveShell.prompts"
548 " object directly.".format(name=name))
548 " object directly.".format(name=name))
549
549
550 # protect against weird cases where self.config may not exist:
550 # protect against weird cases where self.config may not exist:
551
551
552 show_rewritten_input = Bool(True,
552 show_rewritten_input = Bool(True,
553 help="Show rewritten input, e.g. for autocall."
553 help="Show rewritten input, e.g. for autocall."
554 ).tag(config=True)
554 ).tag(config=True)
555
555
556 quiet = Bool(False).tag(config=True)
556 quiet = Bool(False).tag(config=True)
557
557
558 history_length = Integer(10000,
558 history_length = Integer(10000,
559 help='Total length of command history'
559 help='Total length of command history'
560 ).tag(config=True)
560 ).tag(config=True)
561
561
562 history_load_length = Integer(1000, help=
562 history_load_length = Integer(1000, help=
563 """
563 """
564 The number of saved history entries to be loaded
564 The number of saved history entries to be loaded
565 into the history buffer at startup.
565 into the history buffer at startup.
566 """
566 """
567 ).tag(config=True)
567 ).tag(config=True)
568
568
569 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
569 ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
570 default_value='last_expr',
570 default_value='last_expr',
571 help="""
571 help="""
572 'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
572 'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
573 which nodes should be run interactively (displaying output from expressions).
573 which nodes should be run interactively (displaying output from expressions).
574 """
574 """
575 ).tag(config=True)
575 ).tag(config=True)
576
576
577 # TODO: this part of prompt management should be moved to the frontends.
577 # TODO: this part of prompt management should be moved to the frontends.
578 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
578 # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
579 separate_in = SeparateUnicode('\n').tag(config=True)
579 separate_in = SeparateUnicode('\n').tag(config=True)
580 separate_out = SeparateUnicode('').tag(config=True)
580 separate_out = SeparateUnicode('').tag(config=True)
581 separate_out2 = SeparateUnicode('').tag(config=True)
581 separate_out2 = SeparateUnicode('').tag(config=True)
582 wildcards_case_sensitive = Bool(True).tag(config=True)
582 wildcards_case_sensitive = Bool(True).tag(config=True)
583 xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
583 xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
584 default_value='Context',
584 default_value='Context',
585 help="Switch modes for the IPython exception handlers."
585 help="Switch modes for the IPython exception handlers."
586 ).tag(config=True)
586 ).tag(config=True)
587
587
588 # Subcomponents of InteractiveShell
588 # Subcomponents of InteractiveShell
589 alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
589 alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
590 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
590 prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
591 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
591 builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
592 display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
592 display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
593 extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
593 extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
594 payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
594 payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
595 history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
595 history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
596 magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
596 magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
597
597
598 profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
598 profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
599 @property
599 @property
600 def profile(self):
600 def profile(self):
601 if self.profile_dir is not None:
601 if self.profile_dir is not None:
602 name = os.path.basename(self.profile_dir.location)
602 name = os.path.basename(self.profile_dir.location)
603 return name.replace('profile_','')
603 return name.replace('profile_','')
604
604
605
605
606 # Private interface
606 # Private interface
607 _post_execute = Dict()
607 _post_execute = Dict()
608
608
609 # Tracks any GUI loop loaded for pylab
609 # Tracks any GUI loop loaded for pylab
610 pylab_gui_select = None
610 pylab_gui_select = None
611
611
612 last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
612 last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
613
613
614 last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
614 last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
615
615
616 def __init__(self, ipython_dir=None, profile_dir=None,
616 def __init__(self, ipython_dir=None, profile_dir=None,
617 user_module=None, user_ns=None,
617 user_module=None, user_ns=None,
618 custom_exceptions=((), None), **kwargs):
618 custom_exceptions=((), None), **kwargs):
619
619
620 # This is where traits with a config_key argument are updated
620 # This is where traits with a config_key argument are updated
621 # from the values on config.
621 # from the values on config.
622 super(InteractiveShell, self).__init__(**kwargs)
622 super(InteractiveShell, self).__init__(**kwargs)
623 if 'PromptManager' in self.config:
623 if 'PromptManager' in self.config:
624 warn('As of IPython 5.0 `PromptManager` config will have no effect'
624 warn('As of IPython 5.0 `PromptManager` config will have no effect'
625 ' and has been replaced by TerminalInteractiveShell.prompts_class')
625 ' and has been replaced by TerminalInteractiveShell.prompts_class')
626 self.configurables = [self]
626 self.configurables = [self]
627
627
628 # These are relatively independent and stateless
628 # These are relatively independent and stateless
629 self.init_ipython_dir(ipython_dir)
629 self.init_ipython_dir(ipython_dir)
630 self.init_profile_dir(profile_dir)
630 self.init_profile_dir(profile_dir)
631 self.init_instance_attrs()
631 self.init_instance_attrs()
632 self.init_environment()
632 self.init_environment()
633
633
634 # Check if we're in a virtualenv, and set up sys.path.
634 # Check if we're in a virtualenv, and set up sys.path.
635 self.init_virtualenv()
635 self.init_virtualenv()
636
636
637 # Create namespaces (user_ns, user_global_ns, etc.)
637 # Create namespaces (user_ns, user_global_ns, etc.)
638 self.init_create_namespaces(user_module, user_ns)
638 self.init_create_namespaces(user_module, user_ns)
639 # This has to be done after init_create_namespaces because it uses
639 # This has to be done after init_create_namespaces because it uses
640 # something in self.user_ns, but before init_sys_modules, which
640 # something in self.user_ns, but before init_sys_modules, which
641 # is the first thing to modify sys.
641 # is the first thing to modify sys.
642 # TODO: When we override sys.stdout and sys.stderr before this class
642 # TODO: When we override sys.stdout and sys.stderr before this class
643 # is created, we are saving the overridden ones here. Not sure if this
643 # is created, we are saving the overridden ones here. Not sure if this
644 # is what we want to do.
644 # is what we want to do.
645 self.save_sys_module_state()
645 self.save_sys_module_state()
646 self.init_sys_modules()
646 self.init_sys_modules()
647
647
648 # While we're trying to have each part of the code directly access what
648 # While we're trying to have each part of the code directly access what
649 # it needs without keeping redundant references to objects, we have too
649 # it needs without keeping redundant references to objects, we have too
650 # much legacy code that expects ip.db to exist.
650 # much legacy code that expects ip.db to exist.
651 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
651 self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
652
652
653 self.init_history()
653 self.init_history()
654 self.init_encoding()
654 self.init_encoding()
655 self.init_prefilter()
655 self.init_prefilter()
656
656
657 self.init_syntax_highlighting()
657 self.init_syntax_highlighting()
658 self.init_hooks()
658 self.init_hooks()
659 self.init_events()
659 self.init_events()
660 self.init_pushd_popd_magic()
660 self.init_pushd_popd_magic()
661 self.init_user_ns()
661 self.init_user_ns()
662 self.init_logger()
662 self.init_logger()
663 self.init_builtins()
663 self.init_builtins()
664
664
665 # The following was in post_config_initialization
665 # The following was in post_config_initialization
666 self.init_inspector()
666 self.init_inspector()
667 self.raw_input_original = input
667 self.raw_input_original = input
668 self.init_completer()
668 self.init_completer()
669 # TODO: init_io() needs to happen before init_traceback handlers
669 # TODO: init_io() needs to happen before init_traceback handlers
670 # because the traceback handlers hardcode the stdout/stderr streams.
670 # because the traceback handlers hardcode the stdout/stderr streams.
671 # This logic in in debugger.Pdb and should eventually be changed.
671 # This logic in in debugger.Pdb and should eventually be changed.
672 self.init_io()
672 self.init_io()
673 self.init_traceback_handlers(custom_exceptions)
673 self.init_traceback_handlers(custom_exceptions)
674 self.init_prompts()
674 self.init_prompts()
675 self.init_display_formatter()
675 self.init_display_formatter()
676 self.init_display_pub()
676 self.init_display_pub()
677 self.init_data_pub()
677 self.init_data_pub()
678 self.init_displayhook()
678 self.init_displayhook()
679 self.init_magics()
679 self.init_magics()
680 self.init_alias()
680 self.init_alias()
681 self.init_logstart()
681 self.init_logstart()
682 self.init_pdb()
682 self.init_pdb()
683 self.init_extension_manager()
683 self.init_extension_manager()
684 self.init_payload()
684 self.init_payload()
685 self.init_deprecation_warnings()
685 self.init_deprecation_warnings()
686 self.hooks.late_startup_hook()
686 self.hooks.late_startup_hook()
687 self.events.trigger('shell_initialized', self)
687 self.events.trigger('shell_initialized', self)
688 atexit.register(self.atexit_operations)
688 atexit.register(self.atexit_operations)
689
689
690 def get_ipython(self):
690 def get_ipython(self):
691 """Return the currently running IPython instance."""
691 """Return the currently running IPython instance."""
692 return self
692 return self
693
693
694 #-------------------------------------------------------------------------
694 #-------------------------------------------------------------------------
695 # Trait changed handlers
695 # Trait changed handlers
696 #-------------------------------------------------------------------------
696 #-------------------------------------------------------------------------
697 @observe('ipython_dir')
697 @observe('ipython_dir')
698 def _ipython_dir_changed(self, change):
698 def _ipython_dir_changed(self, change):
699 ensure_dir_exists(change['new'])
699 ensure_dir_exists(change['new'])
700
700
701 def set_autoindent(self,value=None):
701 def set_autoindent(self,value=None):
702 """Set the autoindent flag.
702 """Set the autoindent flag.
703
703
704 If called with no arguments, it acts as a toggle."""
704 If called with no arguments, it acts as a toggle."""
705 if value is None:
705 if value is None:
706 self.autoindent = not self.autoindent
706 self.autoindent = not self.autoindent
707 else:
707 else:
708 self.autoindent = value
708 self.autoindent = value
709
709
710 #-------------------------------------------------------------------------
710 #-------------------------------------------------------------------------
711 # init_* methods called by __init__
711 # init_* methods called by __init__
712 #-------------------------------------------------------------------------
712 #-------------------------------------------------------------------------
713
713
714 def init_ipython_dir(self, ipython_dir):
714 def init_ipython_dir(self, ipython_dir):
715 if ipython_dir is not None:
715 if ipython_dir is not None:
716 self.ipython_dir = ipython_dir
716 self.ipython_dir = ipython_dir
717 return
717 return
718
718
719 self.ipython_dir = get_ipython_dir()
719 self.ipython_dir = get_ipython_dir()
720
720
721 def init_profile_dir(self, profile_dir):
721 def init_profile_dir(self, profile_dir):
722 if profile_dir is not None:
722 if profile_dir is not None:
723 self.profile_dir = profile_dir
723 self.profile_dir = profile_dir
724 return
724 return
725 self.profile_dir =\
725 self.profile_dir =\
726 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
726 ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
727
727
728 def init_instance_attrs(self):
728 def init_instance_attrs(self):
729 self.more = False
729 self.more = False
730
730
731 # command compiler
731 # command compiler
732 self.compile = CachingCompiler()
732 self.compile = CachingCompiler()
733
733
734 # Make an empty namespace, which extension writers can rely on both
734 # Make an empty namespace, which extension writers can rely on both
735 # existing and NEVER being used by ipython itself. This gives them a
735 # existing and NEVER being used by ipython itself. This gives them a
736 # convenient location for storing additional information and state
736 # convenient location for storing additional information and state
737 # their extensions may require, without fear of collisions with other
737 # their extensions may require, without fear of collisions with other
738 # ipython names that may develop later.
738 # ipython names that may develop later.
739 self.meta = Struct()
739 self.meta = Struct()
740
740
741 # Temporary files used for various purposes. Deleted at exit.
741 # Temporary files used for various purposes. Deleted at exit.
742 self.tempfiles = []
742 self.tempfiles = []
743 self.tempdirs = []
743 self.tempdirs = []
744
744
745 # keep track of where we started running (mainly for crash post-mortem)
745 # keep track of where we started running (mainly for crash post-mortem)
746 # This is not being used anywhere currently.
746 # This is not being used anywhere currently.
747 self.starting_dir = os.getcwd()
747 self.starting_dir = os.getcwd()
748
748
749 # Indentation management
749 # Indentation management
750 self.indent_current_nsp = 0
750 self.indent_current_nsp = 0
751
751
752 # Dict to track post-execution functions that have been registered
752 # Dict to track post-execution functions that have been registered
753 self._post_execute = {}
753 self._post_execute = {}
754
754
755 def init_environment(self):
755 def init_environment(self):
756 """Any changes we need to make to the user's environment."""
756 """Any changes we need to make to the user's environment."""
757 pass
757 pass
758
758
759 def init_encoding(self):
759 def init_encoding(self):
760 # Get system encoding at startup time. Certain terminals (like Emacs
760 # Get system encoding at startup time. Certain terminals (like Emacs
761 # under Win32 have it set to None, and we need to have a known valid
761 # under Win32 have it set to None, and we need to have a known valid
762 # encoding to use in the raw_input() method
762 # encoding to use in the raw_input() method
763 try:
763 try:
764 self.stdin_encoding = sys.stdin.encoding or 'ascii'
764 self.stdin_encoding = sys.stdin.encoding or 'ascii'
765 except AttributeError:
765 except AttributeError:
766 self.stdin_encoding = 'ascii'
766 self.stdin_encoding = 'ascii'
767
767
768
768
769 @observe('colors')
769 @observe('colors')
770 def init_syntax_highlighting(self, changes=None):
770 def init_syntax_highlighting(self, changes=None):
771 # Python source parser/formatter for syntax highlighting
771 # Python source parser/formatter for syntax highlighting
772 pyformat = PyColorize.Parser(style=self.colors, parent=self).format
772 pyformat = PyColorize.Parser(style=self.colors, parent=self).format
773 self.pycolorize = lambda src: pyformat(src,'str')
773 self.pycolorize = lambda src: pyformat(src,'str')
774
774
775 def refresh_style(self):
775 def refresh_style(self):
776 # No-op here, used in subclass
776 # No-op here, used in subclass
777 pass
777 pass
778
778
779 def init_pushd_popd_magic(self):
779 def init_pushd_popd_magic(self):
780 # for pushd/popd management
780 # for pushd/popd management
781 self.home_dir = get_home_dir()
781 self.home_dir = get_home_dir()
782
782
783 self.dir_stack = []
783 self.dir_stack = []
784
784
785 def init_logger(self):
785 def init_logger(self):
786 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
786 self.logger = Logger(self.home_dir, logfname='ipython_log.py',
787 logmode='rotate')
787 logmode='rotate')
788
788
789 def init_logstart(self):
789 def init_logstart(self):
790 """Initialize logging in case it was requested at the command line.
790 """Initialize logging in case it was requested at the command line.
791 """
791 """
792 if self.logappend:
792 if self.logappend:
793 self.magic('logstart %s append' % self.logappend)
793 self.magic('logstart %s append' % self.logappend)
794 elif self.logfile:
794 elif self.logfile:
795 self.magic('logstart %s' % self.logfile)
795 self.magic('logstart %s' % self.logfile)
796 elif self.logstart:
796 elif self.logstart:
797 self.magic('logstart')
797 self.magic('logstart')
798
798
799 def init_deprecation_warnings(self):
799 def init_deprecation_warnings(self):
800 """
800 """
801 register default filter for deprecation warning.
801 register default filter for deprecation warning.
802
802
803 This will allow deprecation warning of function used interactively to show
803 This will allow deprecation warning of function used interactively to show
804 warning to users, and still hide deprecation warning from libraries import.
804 warning to users, and still hide deprecation warning from libraries import.
805 """
805 """
806 if sys.version_info < (3,7):
806 if sys.version_info < (3,7):
807 warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
807 warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
808
808
809
809
810 def init_builtins(self):
810 def init_builtins(self):
811 # A single, static flag that we set to True. Its presence indicates
811 # A single, static flag that we set to True. Its presence indicates
812 # that an IPython shell has been created, and we make no attempts at
812 # that an IPython shell has been created, and we make no attempts at
813 # removing on exit or representing the existence of more than one
813 # removing on exit or representing the existence of more than one
814 # IPython at a time.
814 # IPython at a time.
815 builtin_mod.__dict__['__IPYTHON__'] = True
815 builtin_mod.__dict__['__IPYTHON__'] = True
816 builtin_mod.__dict__['display'] = display
816 builtin_mod.__dict__['display'] = display
817
817
818 self.builtin_trap = BuiltinTrap(shell=self)
818 self.builtin_trap = BuiltinTrap(shell=self)
819
819
820 @observe('colors')
820 @observe('colors')
821 def init_inspector(self, changes=None):
821 def init_inspector(self, changes=None):
822 # Object inspector
822 # Object inspector
823 self.inspector = oinspect.Inspector(oinspect.InspectColors,
823 self.inspector = oinspect.Inspector(oinspect.InspectColors,
824 PyColorize.ANSICodeColors,
824 PyColorize.ANSICodeColors,
825 self.colors,
825 self.colors,
826 self.object_info_string_level)
826 self.object_info_string_level)
827
827
828 def init_io(self):
828 def init_io(self):
829 # This will just use sys.stdout and sys.stderr. If you want to
829 # This will just use sys.stdout and sys.stderr. If you want to
830 # override sys.stdout and sys.stderr themselves, you need to do that
830 # override sys.stdout and sys.stderr themselves, you need to do that
831 # *before* instantiating this class, because io holds onto
831 # *before* instantiating this class, because io holds onto
832 # references to the underlying streams.
832 # references to the underlying streams.
833 # io.std* are deprecated, but don't show our own deprecation warnings
833 # io.std* are deprecated, but don't show our own deprecation warnings
834 # during initialization of the deprecated API.
834 # during initialization of the deprecated API.
835 with warnings.catch_warnings():
835 with warnings.catch_warnings():
836 warnings.simplefilter('ignore', DeprecationWarning)
836 warnings.simplefilter('ignore', DeprecationWarning)
837 io.stdout = io.IOStream(sys.stdout)
837 io.stdout = io.IOStream(sys.stdout)
838 io.stderr = io.IOStream(sys.stderr)
838 io.stderr = io.IOStream(sys.stderr)
839
839
840 def init_prompts(self):
840 def init_prompts(self):
841 # Set system prompts, so that scripts can decide if they are running
841 # Set system prompts, so that scripts can decide if they are running
842 # interactively.
842 # interactively.
843 sys.ps1 = 'In : '
843 sys.ps1 = 'In : '
844 sys.ps2 = '...: '
844 sys.ps2 = '...: '
845 sys.ps3 = 'Out: '
845 sys.ps3 = 'Out: '
846
846
847 def init_display_formatter(self):
847 def init_display_formatter(self):
848 self.display_formatter = DisplayFormatter(parent=self)
848 self.display_formatter = DisplayFormatter(parent=self)
849 self.configurables.append(self.display_formatter)
849 self.configurables.append(self.display_formatter)
850
850
851 def init_display_pub(self):
851 def init_display_pub(self):
852 self.display_pub = self.display_pub_class(parent=self)
852 self.display_pub = self.display_pub_class(parent=self)
853 self.configurables.append(self.display_pub)
853 self.configurables.append(self.display_pub)
854
854
855 def init_data_pub(self):
855 def init_data_pub(self):
856 if not self.data_pub_class:
856 if not self.data_pub_class:
857 self.data_pub = None
857 self.data_pub = None
858 return
858 return
859 self.data_pub = self.data_pub_class(parent=self)
859 self.data_pub = self.data_pub_class(parent=self)
860 self.configurables.append(self.data_pub)
860 self.configurables.append(self.data_pub)
861
861
862 def init_displayhook(self):
862 def init_displayhook(self):
863 # Initialize displayhook, set in/out prompts and printing system
863 # Initialize displayhook, set in/out prompts and printing system
864 self.displayhook = self.displayhook_class(
864 self.displayhook = self.displayhook_class(
865 parent=self,
865 parent=self,
866 shell=self,
866 shell=self,
867 cache_size=self.cache_size,
867 cache_size=self.cache_size,
868 )
868 )
869 self.configurables.append(self.displayhook)
869 self.configurables.append(self.displayhook)
870 # This is a context manager that installs/revmoes the displayhook at
870 # This is a context manager that installs/revmoes the displayhook at
871 # the appropriate time.
871 # the appropriate time.
872 self.display_trap = DisplayTrap(hook=self.displayhook)
872 self.display_trap = DisplayTrap(hook=self.displayhook)
873
873
874 def init_virtualenv(self):
874 def init_virtualenv(self):
875 """Add a virtualenv to sys.path so the user can import modules from it.
875 """Add a virtualenv to sys.path so the user can import modules from it.
876 This isn't perfect: it doesn't use the Python interpreter with which the
876 This isn't perfect: it doesn't use the Python interpreter with which the
877 virtualenv was built, and it ignores the --no-site-packages option. A
877 virtualenv was built, and it ignores the --no-site-packages option. A
878 warning will appear suggesting the user installs IPython in the
878 warning will appear suggesting the user installs IPython in the
879 virtualenv, but for many cases, it probably works well enough.
879 virtualenv, but for many cases, it probably works well enough.
880
880
881 Adapted from code snippets online.
881 Adapted from code snippets online.
882
882
883 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
883 http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
884 """
884 """
885 if 'VIRTUAL_ENV' not in os.environ:
885 if 'VIRTUAL_ENV' not in os.environ:
886 # Not in a virtualenv
886 # Not in a virtualenv
887 return
887 return
888
888
889 p = os.path.normcase(sys.executable)
889 p = os.path.normcase(sys.executable)
890 p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
890 p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
891
891
892 # executable path should end like /bin/python or \\scripts\\python.exe
892 # executable path should end like /bin/python or \\scripts\\python.exe
893 p_exe_up2 = os.path.dirname(os.path.dirname(p))
893 p_exe_up2 = os.path.dirname(os.path.dirname(p))
894 if p_exe_up2 and os.path.exists(p_venv) and os.path.samefile(p_exe_up2, p_venv):
894 if p_exe_up2 and os.path.exists(p_venv) and os.path.samefile(p_exe_up2, p_venv):
895 # Our exe is inside the virtualenv, don't need to do anything.
895 # Our exe is inside the virtualenv, don't need to do anything.
896 return
896 return
897
897
898 # fallback venv detection:
898 # fallback venv detection:
899 # stdlib venv may symlink sys.executable, so we can't use realpath.
899 # stdlib venv may symlink sys.executable, so we can't use realpath.
900 # but others can symlink *to* the venv Python, so we can't just use sys.executable.
900 # but others can symlink *to* the venv Python, so we can't just use sys.executable.
901 # So we just check every item in the symlink tree (generally <= 3)
901 # So we just check every item in the symlink tree (generally <= 3)
902 paths = [p]
902 paths = [p]
903 while os.path.islink(p):
903 while os.path.islink(p):
904 p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
904 p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
905 paths.append(p)
905 paths.append(p)
906
906
907 # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
907 # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
908 if p_venv.startswith('\\cygdrive'):
908 if p_venv.startswith('\\cygdrive'):
909 p_venv = p_venv[11:]
909 p_venv = p_venv[11:]
910 elif len(p_venv) >= 2 and p_venv[1] == ':':
910 elif len(p_venv) >= 2 and p_venv[1] == ':':
911 p_venv = p_venv[2:]
911 p_venv = p_venv[2:]
912
912
913 if any(p_venv in p for p in paths):
913 if any(p_venv in p for p in paths):
914 # Running properly in the virtualenv, don't need to do anything
914 # Running properly in the virtualenv, don't need to do anything
915 return
915 return
916
916
917 warn("Attempting to work in a virtualenv. If you encounter problems, please "
917 warn("Attempting to work in a virtualenv. If you encounter problems, please "
918 "install IPython inside the virtualenv.")
918 "install IPython inside the virtualenv.")
919 if sys.platform == "win32":
919 if sys.platform == "win32":
920 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
920 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages')
921 else:
921 else:
922 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
922 virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
923 'python%d.%d' % sys.version_info[:2], 'site-packages')
923 'python%d.%d' % sys.version_info[:2], 'site-packages')
924
924
925 import site
925 import site
926 sys.path.insert(0, virtual_env)
926 sys.path.insert(0, virtual_env)
927 site.addsitedir(virtual_env)
927 site.addsitedir(virtual_env)
928
928
929 #-------------------------------------------------------------------------
929 #-------------------------------------------------------------------------
930 # Things related to injections into the sys module
930 # Things related to injections into the sys module
931 #-------------------------------------------------------------------------
931 #-------------------------------------------------------------------------
932
932
933 def save_sys_module_state(self):
933 def save_sys_module_state(self):
934 """Save the state of hooks in the sys module.
934 """Save the state of hooks in the sys module.
935
935
936 This has to be called after self.user_module is created.
936 This has to be called after self.user_module is created.
937 """
937 """
938 self._orig_sys_module_state = {'stdin': sys.stdin,
938 self._orig_sys_module_state = {'stdin': sys.stdin,
939 'stdout': sys.stdout,
939 'stdout': sys.stdout,
940 'stderr': sys.stderr,
940 'stderr': sys.stderr,
941 'excepthook': sys.excepthook}
941 'excepthook': sys.excepthook}
942 self._orig_sys_modules_main_name = self.user_module.__name__
942 self._orig_sys_modules_main_name = self.user_module.__name__
943 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
943 self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
944
944
945 def restore_sys_module_state(self):
945 def restore_sys_module_state(self):
946 """Restore the state of the sys module."""
946 """Restore the state of the sys module."""
947 try:
947 try:
948 for k, v in self._orig_sys_module_state.items():
948 for k, v in self._orig_sys_module_state.items():
949 setattr(sys, k, v)
949 setattr(sys, k, v)
950 except AttributeError:
950 except AttributeError:
951 pass
951 pass
952 # Reset what what done in self.init_sys_modules
952 # Reset what what done in self.init_sys_modules
953 if self._orig_sys_modules_main_mod is not None:
953 if self._orig_sys_modules_main_mod is not None:
954 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
954 sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
955
955
956 #-------------------------------------------------------------------------
956 #-------------------------------------------------------------------------
957 # Things related to the banner
957 # Things related to the banner
958 #-------------------------------------------------------------------------
958 #-------------------------------------------------------------------------
959
959
960 @property
960 @property
961 def banner(self):
961 def banner(self):
962 banner = self.banner1
962 banner = self.banner1
963 if self.profile and self.profile != 'default':
963 if self.profile and self.profile != 'default':
964 banner += '\nIPython profile: %s\n' % self.profile
964 banner += '\nIPython profile: %s\n' % self.profile
965 if self.banner2:
965 if self.banner2:
966 banner += '\n' + self.banner2
966 banner += '\n' + self.banner2
967 return banner
967 return banner
968
968
969 def show_banner(self, banner=None):
969 def show_banner(self, banner=None):
970 if banner is None:
970 if banner is None:
971 banner = self.banner
971 banner = self.banner
972 sys.stdout.write(banner)
972 sys.stdout.write(banner)
973
973
974 #-------------------------------------------------------------------------
974 #-------------------------------------------------------------------------
975 # Things related to hooks
975 # Things related to hooks
976 #-------------------------------------------------------------------------
976 #-------------------------------------------------------------------------
977
977
978 def init_hooks(self):
978 def init_hooks(self):
979 # hooks holds pointers used for user-side customizations
979 # hooks holds pointers used for user-side customizations
980 self.hooks = Struct()
980 self.hooks = Struct()
981
981
982 self.strdispatchers = {}
982 self.strdispatchers = {}
983
983
984 # Set all default hooks, defined in the IPython.hooks module.
984 # Set all default hooks, defined in the IPython.hooks module.
985 hooks = IPython.core.hooks
985 hooks = IPython.core.hooks
986 for hook_name in hooks.__all__:
986 for hook_name in hooks.__all__:
987 # default hooks have priority 100, i.e. low; user hooks should have
987 # default hooks have priority 100, i.e. low; user hooks should have
988 # 0-100 priority
988 # 0-100 priority
989 self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
989 self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
990
990
991 if self.display_page:
991 if self.display_page:
992 self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
992 self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
993
993
994 def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
994 def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
995 _warn_deprecated=True):
995 _warn_deprecated=True):
996 """set_hook(name,hook) -> sets an internal IPython hook.
996 """set_hook(name,hook) -> sets an internal IPython hook.
997
997
998 IPython exposes some of its internal API as user-modifiable hooks. By
998 IPython exposes some of its internal API as user-modifiable hooks. By
999 adding your function to one of these hooks, you can modify IPython's
999 adding your function to one of these hooks, you can modify IPython's
1000 behavior to call at runtime your own routines."""
1000 behavior to call at runtime your own routines."""
1001
1001
1002 # At some point in the future, this should validate the hook before it
1002 # At some point in the future, this should validate the hook before it
1003 # accepts it. Probably at least check that the hook takes the number
1003 # accepts it. Probably at least check that the hook takes the number
1004 # of args it's supposed to.
1004 # of args it's supposed to.
1005
1005
1006 f = types.MethodType(hook,self)
1006 f = types.MethodType(hook,self)
1007
1007
1008 # check if the hook is for strdispatcher first
1008 # check if the hook is for strdispatcher first
1009 if str_key is not None:
1009 if str_key is not None:
1010 sdp = self.strdispatchers.get(name, StrDispatch())
1010 sdp = self.strdispatchers.get(name, StrDispatch())
1011 sdp.add_s(str_key, f, priority )
1011 sdp.add_s(str_key, f, priority )
1012 self.strdispatchers[name] = sdp
1012 self.strdispatchers[name] = sdp
1013 return
1013 return
1014 if re_key is not None:
1014 if re_key is not None:
1015 sdp = self.strdispatchers.get(name, StrDispatch())
1015 sdp = self.strdispatchers.get(name, StrDispatch())
1016 sdp.add_re(re.compile(re_key), f, priority )
1016 sdp.add_re(re.compile(re_key), f, priority )
1017 self.strdispatchers[name] = sdp
1017 self.strdispatchers[name] = sdp
1018 return
1018 return
1019
1019
1020 dp = getattr(self.hooks, name, None)
1020 dp = getattr(self.hooks, name, None)
1021 if name not in IPython.core.hooks.__all__:
1021 if name not in IPython.core.hooks.__all__:
1022 print("Warning! Hook '%s' is not one of %s" % \
1022 print("Warning! Hook '%s' is not one of %s" % \
1023 (name, IPython.core.hooks.__all__ ))
1023 (name, IPython.core.hooks.__all__ ))
1024
1024
1025 if _warn_deprecated and (name in IPython.core.hooks.deprecated):
1025 if _warn_deprecated and (name in IPython.core.hooks.deprecated):
1026 alternative = IPython.core.hooks.deprecated[name]
1026 alternative = IPython.core.hooks.deprecated[name]
1027 warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
1027 warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
1028
1028
1029 if not dp:
1029 if not dp:
1030 dp = IPython.core.hooks.CommandChainDispatcher()
1030 dp = IPython.core.hooks.CommandChainDispatcher()
1031
1031
1032 try:
1032 try:
1033 dp.add(f,priority)
1033 dp.add(f,priority)
1034 except AttributeError:
1034 except AttributeError:
1035 # it was not commandchain, plain old func - replace
1035 # it was not commandchain, plain old func - replace
1036 dp = f
1036 dp = f
1037
1037
1038 setattr(self.hooks,name, dp)
1038 setattr(self.hooks,name, dp)
1039
1039
1040 #-------------------------------------------------------------------------
1040 #-------------------------------------------------------------------------
1041 # Things related to events
1041 # Things related to events
1042 #-------------------------------------------------------------------------
1042 #-------------------------------------------------------------------------
1043
1043
1044 def init_events(self):
1044 def init_events(self):
1045 self.events = EventManager(self, available_events)
1045 self.events = EventManager(self, available_events)
1046
1046
1047 self.events.register("pre_execute", self._clear_warning_registry)
1047 self.events.register("pre_execute", self._clear_warning_registry)
1048
1048
1049 def register_post_execute(self, func):
1049 def register_post_execute(self, func):
1050 """DEPRECATED: Use ip.events.register('post_run_cell', func)
1050 """DEPRECATED: Use ip.events.register('post_run_cell', func)
1051
1051
1052 Register a function for calling after code execution.
1052 Register a function for calling after code execution.
1053 """
1053 """
1054 warn("ip.register_post_execute is deprecated, use "
1054 warn("ip.register_post_execute is deprecated, use "
1055 "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
1055 "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
1056 self.events.register('post_run_cell', func)
1056 self.events.register('post_run_cell', func)
1057
1057
1058 def _clear_warning_registry(self):
1058 def _clear_warning_registry(self):
1059 # clear the warning registry, so that different code blocks with
1059 # clear the warning registry, so that different code blocks with
1060 # overlapping line number ranges don't cause spurious suppression of
1060 # overlapping line number ranges don't cause spurious suppression of
1061 # warnings (see gh-6611 for details)
1061 # warnings (see gh-6611 for details)
1062 if "__warningregistry__" in self.user_global_ns:
1062 if "__warningregistry__" in self.user_global_ns:
1063 del self.user_global_ns["__warningregistry__"]
1063 del self.user_global_ns["__warningregistry__"]
1064
1064
1065 #-------------------------------------------------------------------------
1065 #-------------------------------------------------------------------------
1066 # Things related to the "main" module
1066 # Things related to the "main" module
1067 #-------------------------------------------------------------------------
1067 #-------------------------------------------------------------------------
1068
1068
1069 def new_main_mod(self, filename, modname):
1069 def new_main_mod(self, filename, modname):
1070 """Return a new 'main' module object for user code execution.
1070 """Return a new 'main' module object for user code execution.
1071
1071
1072 ``filename`` should be the path of the script which will be run in the
1072 ``filename`` should be the path of the script which will be run in the
1073 module. Requests with the same filename will get the same module, with
1073 module. Requests with the same filename will get the same module, with
1074 its namespace cleared.
1074 its namespace cleared.
1075
1075
1076 ``modname`` should be the module name - normally either '__main__' or
1076 ``modname`` should be the module name - normally either '__main__' or
1077 the basename of the file without the extension.
1077 the basename of the file without the extension.
1078
1078
1079 When scripts are executed via %run, we must keep a reference to their
1079 When scripts are executed via %run, we must keep a reference to their
1080 __main__ module around so that Python doesn't
1080 __main__ module around so that Python doesn't
1081 clear it, rendering references to module globals useless.
1081 clear it, rendering references to module globals useless.
1082
1082
1083 This method keeps said reference in a private dict, keyed by the
1083 This method keeps said reference in a private dict, keyed by the
1084 absolute path of the script. This way, for multiple executions of the
1084 absolute path of the script. This way, for multiple executions of the
1085 same script we only keep one copy of the namespace (the last one),
1085 same script we only keep one copy of the namespace (the last one),
1086 thus preventing memory leaks from old references while allowing the
1086 thus preventing memory leaks from old references while allowing the
1087 objects from the last execution to be accessible.
1087 objects from the last execution to be accessible.
1088 """
1088 """
1089 filename = os.path.abspath(filename)
1089 filename = os.path.abspath(filename)
1090 try:
1090 try:
1091 main_mod = self._main_mod_cache[filename]
1091 main_mod = self._main_mod_cache[filename]
1092 except KeyError:
1092 except KeyError:
1093 main_mod = self._main_mod_cache[filename] = types.ModuleType(
1093 main_mod = self._main_mod_cache[filename] = types.ModuleType(
1094 modname,
1094 modname,
1095 doc="Module created for script run in IPython")
1095 doc="Module created for script run in IPython")
1096 else:
1096 else:
1097 main_mod.__dict__.clear()
1097 main_mod.__dict__.clear()
1098 main_mod.__name__ = modname
1098 main_mod.__name__ = modname
1099
1099
1100 main_mod.__file__ = filename
1100 main_mod.__file__ = filename
1101 # It seems pydoc (and perhaps others) needs any module instance to
1101 # It seems pydoc (and perhaps others) needs any module instance to
1102 # implement a __nonzero__ method
1102 # implement a __nonzero__ method
1103 main_mod.__nonzero__ = lambda : True
1103 main_mod.__nonzero__ = lambda : True
1104
1104
1105 return main_mod
1105 return main_mod
1106
1106
1107 def clear_main_mod_cache(self):
1107 def clear_main_mod_cache(self):
1108 """Clear the cache of main modules.
1108 """Clear the cache of main modules.
1109
1109
1110 Mainly for use by utilities like %reset.
1110 Mainly for use by utilities like %reset.
1111
1111
1112 Examples
1112 Examples
1113 --------
1113 --------
1114
1114
1115 In [15]: import IPython
1115 In [15]: import IPython
1116
1116
1117 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
1117 In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
1118
1118
1119 In [17]: len(_ip._main_mod_cache) > 0
1119 In [17]: len(_ip._main_mod_cache) > 0
1120 Out[17]: True
1120 Out[17]: True
1121
1121
1122 In [18]: _ip.clear_main_mod_cache()
1122 In [18]: _ip.clear_main_mod_cache()
1123
1123
1124 In [19]: len(_ip._main_mod_cache) == 0
1124 In [19]: len(_ip._main_mod_cache) == 0
1125 Out[19]: True
1125 Out[19]: True
1126 """
1126 """
1127 self._main_mod_cache.clear()
1127 self._main_mod_cache.clear()
1128
1128
1129 #-------------------------------------------------------------------------
1129 #-------------------------------------------------------------------------
1130 # Things related to debugging
1130 # Things related to debugging
1131 #-------------------------------------------------------------------------
1131 #-------------------------------------------------------------------------
1132
1132
1133 def init_pdb(self):
1133 def init_pdb(self):
1134 # Set calling of pdb on exceptions
1134 # Set calling of pdb on exceptions
1135 # self.call_pdb is a property
1135 # self.call_pdb is a property
1136 self.call_pdb = self.pdb
1136 self.call_pdb = self.pdb
1137
1137
1138 def _get_call_pdb(self):
1138 def _get_call_pdb(self):
1139 return self._call_pdb
1139 return self._call_pdb
1140
1140
1141 def _set_call_pdb(self,val):
1141 def _set_call_pdb(self,val):
1142
1142
1143 if val not in (0,1,False,True):
1143 if val not in (0,1,False,True):
1144 raise ValueError('new call_pdb value must be boolean')
1144 raise ValueError('new call_pdb value must be boolean')
1145
1145
1146 # store value in instance
1146 # store value in instance
1147 self._call_pdb = val
1147 self._call_pdb = val
1148
1148
1149 # notify the actual exception handlers
1149 # notify the actual exception handlers
1150 self.InteractiveTB.call_pdb = val
1150 self.InteractiveTB.call_pdb = val
1151
1151
1152 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
1152 call_pdb = property(_get_call_pdb,_set_call_pdb,None,
1153 'Control auto-activation of pdb at exceptions')
1153 'Control auto-activation of pdb at exceptions')
1154
1154
1155 def debugger(self,force=False):
1155 def debugger(self,force=False):
1156 """Call the pdb debugger.
1156 """Call the pdb debugger.
1157
1157
1158 Keywords:
1158 Keywords:
1159
1159
1160 - force(False): by default, this routine checks the instance call_pdb
1160 - force(False): by default, this routine checks the instance call_pdb
1161 flag and does not actually invoke the debugger if the flag is false.
1161 flag and does not actually invoke the debugger if the flag is false.
1162 The 'force' option forces the debugger to activate even if the flag
1162 The 'force' option forces the debugger to activate even if the flag
1163 is false.
1163 is false.
1164 """
1164 """
1165
1165
1166 if not (force or self.call_pdb):
1166 if not (force or self.call_pdb):
1167 return
1167 return
1168
1168
1169 if not hasattr(sys,'last_traceback'):
1169 if not hasattr(sys,'last_traceback'):
1170 error('No traceback has been produced, nothing to debug.')
1170 error('No traceback has been produced, nothing to debug.')
1171 return
1171 return
1172
1172
1173 self.InteractiveTB.debugger(force=True)
1173 self.InteractiveTB.debugger(force=True)
1174
1174
1175 #-------------------------------------------------------------------------
1175 #-------------------------------------------------------------------------
1176 # Things related to IPython's various namespaces
1176 # Things related to IPython's various namespaces
1177 #-------------------------------------------------------------------------
1177 #-------------------------------------------------------------------------
1178 default_user_namespaces = True
1178 default_user_namespaces = True
1179
1179
1180 def init_create_namespaces(self, user_module=None, user_ns=None):
1180 def init_create_namespaces(self, user_module=None, user_ns=None):
1181 # Create the namespace where the user will operate. user_ns is
1181 # Create the namespace where the user will operate. user_ns is
1182 # normally the only one used, and it is passed to the exec calls as
1182 # normally the only one used, and it is passed to the exec calls as
1183 # the locals argument. But we do carry a user_global_ns namespace
1183 # the locals argument. But we do carry a user_global_ns namespace
1184 # given as the exec 'globals' argument, This is useful in embedding
1184 # given as the exec 'globals' argument, This is useful in embedding
1185 # situations where the ipython shell opens in a context where the
1185 # situations where the ipython shell opens in a context where the
1186 # distinction between locals and globals is meaningful. For
1186 # distinction between locals and globals is meaningful. For
1187 # non-embedded contexts, it is just the same object as the user_ns dict.
1187 # non-embedded contexts, it is just the same object as the user_ns dict.
1188
1188
1189 # FIXME. For some strange reason, __builtins__ is showing up at user
1189 # FIXME. For some strange reason, __builtins__ is showing up at user
1190 # level as a dict instead of a module. This is a manual fix, but I
1190 # level as a dict instead of a module. This is a manual fix, but I
1191 # should really track down where the problem is coming from. Alex
1191 # should really track down where the problem is coming from. Alex
1192 # Schmolck reported this problem first.
1192 # Schmolck reported this problem first.
1193
1193
1194 # A useful post by Alex Martelli on this topic:
1194 # A useful post by Alex Martelli on this topic:
1195 # Re: inconsistent value from __builtins__
1195 # Re: inconsistent value from __builtins__
1196 # Von: Alex Martelli <aleaxit@yahoo.com>
1196 # Von: Alex Martelli <aleaxit@yahoo.com>
1197 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
1197 # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
1198 # Gruppen: comp.lang.python
1198 # Gruppen: comp.lang.python
1199
1199
1200 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
1200 # Michael Hohn <hohn@hooknose.lbl.gov> wrote:
1201 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
1201 # > >>> print type(builtin_check.get_global_binding('__builtins__'))
1202 # > <type 'dict'>
1202 # > <type 'dict'>
1203 # > >>> print type(__builtins__)
1203 # > >>> print type(__builtins__)
1204 # > <type 'module'>
1204 # > <type 'module'>
1205 # > Is this difference in return value intentional?
1205 # > Is this difference in return value intentional?
1206
1206
1207 # Well, it's documented that '__builtins__' can be either a dictionary
1207 # Well, it's documented that '__builtins__' can be either a dictionary
1208 # or a module, and it's been that way for a long time. Whether it's
1208 # or a module, and it's been that way for a long time. Whether it's
1209 # intentional (or sensible), I don't know. In any case, the idea is
1209 # intentional (or sensible), I don't know. In any case, the idea is
1210 # that if you need to access the built-in namespace directly, you
1210 # that if you need to access the built-in namespace directly, you
1211 # should start with "import __builtin__" (note, no 's') which will
1211 # should start with "import __builtin__" (note, no 's') which will
1212 # definitely give you a module. Yeah, it's somewhat confusing:-(.
1212 # definitely give you a module. Yeah, it's somewhat confusing:-(.
1213
1213
1214 # These routines return a properly built module and dict as needed by
1214 # These routines return a properly built module and dict as needed by
1215 # the rest of the code, and can also be used by extension writers to
1215 # the rest of the code, and can also be used by extension writers to
1216 # generate properly initialized namespaces.
1216 # generate properly initialized namespaces.
1217 if (user_ns is not None) or (user_module is not None):
1217 if (user_ns is not None) or (user_module is not None):
1218 self.default_user_namespaces = False
1218 self.default_user_namespaces = False
1219 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
1219 self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
1220
1220
1221 # A record of hidden variables we have added to the user namespace, so
1221 # A record of hidden variables we have added to the user namespace, so
1222 # we can list later only variables defined in actual interactive use.
1222 # we can list later only variables defined in actual interactive use.
1223 self.user_ns_hidden = {}
1223 self.user_ns_hidden = {}
1224
1224
1225 # Now that FakeModule produces a real module, we've run into a nasty
1225 # Now that FakeModule produces a real module, we've run into a nasty
1226 # problem: after script execution (via %run), the module where the user
1226 # problem: after script execution (via %run), the module where the user
1227 # code ran is deleted. Now that this object is a true module (needed
1227 # code ran is deleted. Now that this object is a true module (needed
1228 # so doctest and other tools work correctly), the Python module
1228 # so doctest and other tools work correctly), the Python module
1229 # teardown mechanism runs over it, and sets to None every variable
1229 # teardown mechanism runs over it, and sets to None every variable
1230 # present in that module. Top-level references to objects from the
1230 # present in that module. Top-level references to objects from the
1231 # script survive, because the user_ns is updated with them. However,
1231 # script survive, because the user_ns is updated with them. However,
1232 # calling functions defined in the script that use other things from
1232 # calling functions defined in the script that use other things from
1233 # the script will fail, because the function's closure had references
1233 # the script will fail, because the function's closure had references
1234 # to the original objects, which are now all None. So we must protect
1234 # to the original objects, which are now all None. So we must protect
1235 # these modules from deletion by keeping a cache.
1235 # these modules from deletion by keeping a cache.
1236 #
1236 #
1237 # To avoid keeping stale modules around (we only need the one from the
1237 # To avoid keeping stale modules around (we only need the one from the
1238 # last run), we use a dict keyed with the full path to the script, so
1238 # last run), we use a dict keyed with the full path to the script, so
1239 # only the last version of the module is held in the cache. Note,
1239 # only the last version of the module is held in the cache. Note,
1240 # however, that we must cache the module *namespace contents* (their
1240 # however, that we must cache the module *namespace contents* (their
1241 # __dict__). Because if we try to cache the actual modules, old ones
1241 # __dict__). Because if we try to cache the actual modules, old ones
1242 # (uncached) could be destroyed while still holding references (such as
1242 # (uncached) could be destroyed while still holding references (such as
1243 # those held by GUI objects that tend to be long-lived)>
1243 # those held by GUI objects that tend to be long-lived)>
1244 #
1244 #
1245 # The %reset command will flush this cache. See the cache_main_mod()
1245 # The %reset command will flush this cache. See the cache_main_mod()
1246 # and clear_main_mod_cache() methods for details on use.
1246 # and clear_main_mod_cache() methods for details on use.
1247
1247
1248 # This is the cache used for 'main' namespaces
1248 # This is the cache used for 'main' namespaces
1249 self._main_mod_cache = {}
1249 self._main_mod_cache = {}
1250
1250
1251 # A table holding all the namespaces IPython deals with, so that
1251 # A table holding all the namespaces IPython deals with, so that
1252 # introspection facilities can search easily.
1252 # introspection facilities can search easily.
1253 self.ns_table = {'user_global':self.user_module.__dict__,
1253 self.ns_table = {'user_global':self.user_module.__dict__,
1254 'user_local':self.user_ns,
1254 'user_local':self.user_ns,
1255 'builtin':builtin_mod.__dict__
1255 'builtin':builtin_mod.__dict__
1256 }
1256 }
1257
1257
1258 @property
1258 @property
1259 def user_global_ns(self):
1259 def user_global_ns(self):
1260 return self.user_module.__dict__
1260 return self.user_module.__dict__
1261
1261
1262 def prepare_user_module(self, user_module=None, user_ns=None):
1262 def prepare_user_module(self, user_module=None, user_ns=None):
1263 """Prepare the module and namespace in which user code will be run.
1263 """Prepare the module and namespace in which user code will be run.
1264
1264
1265 When IPython is started normally, both parameters are None: a new module
1265 When IPython is started normally, both parameters are None: a new module
1266 is created automatically, and its __dict__ used as the namespace.
1266 is created automatically, and its __dict__ used as the namespace.
1267
1267
1268 If only user_module is provided, its __dict__ is used as the namespace.
1268 If only user_module is provided, its __dict__ is used as the namespace.
1269 If only user_ns is provided, a dummy module is created, and user_ns
1269 If only user_ns is provided, a dummy module is created, and user_ns
1270 becomes the global namespace. If both are provided (as they may be
1270 becomes the global namespace. If both are provided (as they may be
1271 when embedding), user_ns is the local namespace, and user_module
1271 when embedding), user_ns is the local namespace, and user_module
1272 provides the global namespace.
1272 provides the global namespace.
1273
1273
1274 Parameters
1274 Parameters
1275 ----------
1275 ----------
1276 user_module : module, optional
1276 user_module : module, optional
1277 The current user module in which IPython is being run. If None,
1277 The current user module in which IPython is being run. If None,
1278 a clean module will be created.
1278 a clean module will be created.
1279 user_ns : dict, optional
1279 user_ns : dict, optional
1280 A namespace in which to run interactive commands.
1280 A namespace in which to run interactive commands.
1281
1281
1282 Returns
1282 Returns
1283 -------
1283 -------
1284 A tuple of user_module and user_ns, each properly initialised.
1284 A tuple of user_module and user_ns, each properly initialised.
1285 """
1285 """
1286 if user_module is None and user_ns is not None:
1286 if user_module is None and user_ns is not None:
1287 user_ns.setdefault("__name__", "__main__")
1287 user_ns.setdefault("__name__", "__main__")
1288 user_module = DummyMod()
1288 user_module = DummyMod()
1289 user_module.__dict__ = user_ns
1289 user_module.__dict__ = user_ns
1290
1290
1291 if user_module is None:
1291 if user_module is None:
1292 user_module = types.ModuleType("__main__",
1292 user_module = types.ModuleType("__main__",
1293 doc="Automatically created module for IPython interactive environment")
1293 doc="Automatically created module for IPython interactive environment")
1294
1294
1295 # We must ensure that __builtin__ (without the final 's') is always
1295 # We must ensure that __builtin__ (without the final 's') is always
1296 # available and pointing to the __builtin__ *module*. For more details:
1296 # available and pointing to the __builtin__ *module*. For more details:
1297 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1297 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1298 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1298 user_module.__dict__.setdefault('__builtin__', builtin_mod)
1299 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1299 user_module.__dict__.setdefault('__builtins__', builtin_mod)
1300
1300
1301 if user_ns is None:
1301 if user_ns is None:
1302 user_ns = user_module.__dict__
1302 user_ns = user_module.__dict__
1303
1303
1304 return user_module, user_ns
1304 return user_module, user_ns
1305
1305
1306 def init_sys_modules(self):
1306 def init_sys_modules(self):
1307 # We need to insert into sys.modules something that looks like a
1307 # We need to insert into sys.modules something that looks like a
1308 # module but which accesses the IPython namespace, for shelve and
1308 # module but which accesses the IPython namespace, for shelve and
1309 # pickle to work interactively. Normally they rely on getting
1309 # pickle to work interactively. Normally they rely on getting
1310 # everything out of __main__, but for embedding purposes each IPython
1310 # everything out of __main__, but for embedding purposes each IPython
1311 # instance has its own private namespace, so we can't go shoving
1311 # instance has its own private namespace, so we can't go shoving
1312 # everything into __main__.
1312 # everything into __main__.
1313
1313
1314 # note, however, that we should only do this for non-embedded
1314 # note, however, that we should only do this for non-embedded
1315 # ipythons, which really mimic the __main__.__dict__ with their own
1315 # ipythons, which really mimic the __main__.__dict__ with their own
1316 # namespace. Embedded instances, on the other hand, should not do
1316 # namespace. Embedded instances, on the other hand, should not do
1317 # this because they need to manage the user local/global namespaces
1317 # this because they need to manage the user local/global namespaces
1318 # only, but they live within a 'normal' __main__ (meaning, they
1318 # only, but they live within a 'normal' __main__ (meaning, they
1319 # shouldn't overtake the execution environment of the script they're
1319 # shouldn't overtake the execution environment of the script they're
1320 # embedded in).
1320 # embedded in).
1321
1321
1322 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1322 # This is overridden in the InteractiveShellEmbed subclass to a no-op.
1323 main_name = self.user_module.__name__
1323 main_name = self.user_module.__name__
1324 sys.modules[main_name] = self.user_module
1324 sys.modules[main_name] = self.user_module
1325
1325
1326 def init_user_ns(self):
1326 def init_user_ns(self):
1327 """Initialize all user-visible namespaces to their minimum defaults.
1327 """Initialize all user-visible namespaces to their minimum defaults.
1328
1328
1329 Certain history lists are also initialized here, as they effectively
1329 Certain history lists are also initialized here, as they effectively
1330 act as user namespaces.
1330 act as user namespaces.
1331
1331
1332 Notes
1332 Notes
1333 -----
1333 -----
1334 All data structures here are only filled in, they are NOT reset by this
1334 All data structures here are only filled in, they are NOT reset by this
1335 method. If they were not empty before, data will simply be added to
1335 method. If they were not empty before, data will simply be added to
1336 them.
1336 them.
1337 """
1337 """
1338 # This function works in two parts: first we put a few things in
1338 # This function works in two parts: first we put a few things in
1339 # user_ns, and we sync that contents into user_ns_hidden so that these
1339 # user_ns, and we sync that contents into user_ns_hidden so that these
1340 # initial variables aren't shown by %who. After the sync, we add the
1340 # initial variables aren't shown by %who. After the sync, we add the
1341 # rest of what we *do* want the user to see with %who even on a new
1341 # rest of what we *do* want the user to see with %who even on a new
1342 # session (probably nothing, so they really only see their own stuff)
1342 # session (probably nothing, so they really only see their own stuff)
1343
1343
1344 # The user dict must *always* have a __builtin__ reference to the
1344 # The user dict must *always* have a __builtin__ reference to the
1345 # Python standard __builtin__ namespace, which must be imported.
1345 # Python standard __builtin__ namespace, which must be imported.
1346 # This is so that certain operations in prompt evaluation can be
1346 # This is so that certain operations in prompt evaluation can be
1347 # reliably executed with builtins. Note that we can NOT use
1347 # reliably executed with builtins. Note that we can NOT use
1348 # __builtins__ (note the 's'), because that can either be a dict or a
1348 # __builtins__ (note the 's'), because that can either be a dict or a
1349 # module, and can even mutate at runtime, depending on the context
1349 # module, and can even mutate at runtime, depending on the context
1350 # (Python makes no guarantees on it). In contrast, __builtin__ is
1350 # (Python makes no guarantees on it). In contrast, __builtin__ is
1351 # always a module object, though it must be explicitly imported.
1351 # always a module object, though it must be explicitly imported.
1352
1352
1353 # For more details:
1353 # For more details:
1354 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1354 # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
1355 ns = {}
1355 ns = {}
1356
1356
1357 # make global variables for user access to the histories
1357 # make global variables for user access to the histories
1358 ns['_ih'] = self.history_manager.input_hist_parsed
1358 ns['_ih'] = self.history_manager.input_hist_parsed
1359 ns['_oh'] = self.history_manager.output_hist
1359 ns['_oh'] = self.history_manager.output_hist
1360 ns['_dh'] = self.history_manager.dir_hist
1360 ns['_dh'] = self.history_manager.dir_hist
1361
1361
1362 # user aliases to input and output histories. These shouldn't show up
1362 # user aliases to input and output histories. These shouldn't show up
1363 # in %who, as they can have very large reprs.
1363 # in %who, as they can have very large reprs.
1364 ns['In'] = self.history_manager.input_hist_parsed
1364 ns['In'] = self.history_manager.input_hist_parsed
1365 ns['Out'] = self.history_manager.output_hist
1365 ns['Out'] = self.history_manager.output_hist
1366
1366
1367 # Store myself as the public api!!!
1367 # Store myself as the public api!!!
1368 ns['get_ipython'] = self.get_ipython
1368 ns['get_ipython'] = self.get_ipython
1369
1369
1370 ns['exit'] = self.exiter
1370 ns['exit'] = self.exiter
1371 ns['quit'] = self.exiter
1371 ns['quit'] = self.exiter
1372
1372
1373 # Sync what we've added so far to user_ns_hidden so these aren't seen
1373 # Sync what we've added so far to user_ns_hidden so these aren't seen
1374 # by %who
1374 # by %who
1375 self.user_ns_hidden.update(ns)
1375 self.user_ns_hidden.update(ns)
1376
1376
1377 # Anything put into ns now would show up in %who. Think twice before
1377 # Anything put into ns now would show up in %who. Think twice before
1378 # putting anything here, as we really want %who to show the user their
1378 # putting anything here, as we really want %who to show the user their
1379 # stuff, not our variables.
1379 # stuff, not our variables.
1380
1380
1381 # Finally, update the real user's namespace
1381 # Finally, update the real user's namespace
1382 self.user_ns.update(ns)
1382 self.user_ns.update(ns)
1383
1383
1384 @property
1384 @property
1385 def all_ns_refs(self):
1385 def all_ns_refs(self):
1386 """Get a list of references to all the namespace dictionaries in which
1386 """Get a list of references to all the namespace dictionaries in which
1387 IPython might store a user-created object.
1387 IPython might store a user-created object.
1388
1388
1389 Note that this does not include the displayhook, which also caches
1389 Note that this does not include the displayhook, which also caches
1390 objects from the output."""
1390 objects from the output."""
1391 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1391 return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
1392 [m.__dict__ for m in self._main_mod_cache.values()]
1392 [m.__dict__ for m in self._main_mod_cache.values()]
1393
1393
1394 def reset(self, new_session=True):
1394 def reset(self, new_session=True):
1395 """Clear all internal namespaces, and attempt to release references to
1395 """Clear all internal namespaces, and attempt to release references to
1396 user objects.
1396 user objects.
1397
1397
1398 If new_session is True, a new history session will be opened.
1398 If new_session is True, a new history session will be opened.
1399 """
1399 """
1400 # Clear histories
1400 # Clear histories
1401 self.history_manager.reset(new_session)
1401 self.history_manager.reset(new_session)
1402 # Reset counter used to index all histories
1402 # Reset counter used to index all histories
1403 if new_session:
1403 if new_session:
1404 self.execution_count = 1
1404 self.execution_count = 1
1405
1405
1406 # Reset last execution result
1406 # Reset last execution result
1407 self.last_execution_succeeded = True
1407 self.last_execution_succeeded = True
1408 self.last_execution_result = None
1408 self.last_execution_result = None
1409
1409
1410 # Flush cached output items
1410 # Flush cached output items
1411 if self.displayhook.do_full_cache:
1411 if self.displayhook.do_full_cache:
1412 self.displayhook.flush()
1412 self.displayhook.flush()
1413
1413
1414 # The main execution namespaces must be cleared very carefully,
1414 # The main execution namespaces must be cleared very carefully,
1415 # skipping the deletion of the builtin-related keys, because doing so
1415 # skipping the deletion of the builtin-related keys, because doing so
1416 # would cause errors in many object's __del__ methods.
1416 # would cause errors in many object's __del__ methods.
1417 if self.user_ns is not self.user_global_ns:
1417 if self.user_ns is not self.user_global_ns:
1418 self.user_ns.clear()
1418 self.user_ns.clear()
1419 ns = self.user_global_ns
1419 ns = self.user_global_ns
1420 drop_keys = set(ns.keys())
1420 drop_keys = set(ns.keys())
1421 drop_keys.discard('__builtin__')
1421 drop_keys.discard('__builtin__')
1422 drop_keys.discard('__builtins__')
1422 drop_keys.discard('__builtins__')
1423 drop_keys.discard('__name__')
1423 drop_keys.discard('__name__')
1424 for k in drop_keys:
1424 for k in drop_keys:
1425 del ns[k]
1425 del ns[k]
1426
1426
1427 self.user_ns_hidden.clear()
1427 self.user_ns_hidden.clear()
1428
1428
1429 # Restore the user namespaces to minimal usability
1429 # Restore the user namespaces to minimal usability
1430 self.init_user_ns()
1430 self.init_user_ns()
1431
1431
1432 # Restore the default and user aliases
1432 # Restore the default and user aliases
1433 self.alias_manager.clear_aliases()
1433 self.alias_manager.clear_aliases()
1434 self.alias_manager.init_aliases()
1434 self.alias_manager.init_aliases()
1435
1435
1436 # Flush the private list of module references kept for script
1436 # Flush the private list of module references kept for script
1437 # execution protection
1437 # execution protection
1438 self.clear_main_mod_cache()
1438 self.clear_main_mod_cache()
1439
1439
1440 def del_var(self, varname, by_name=False):
1440 def del_var(self, varname, by_name=False):
1441 """Delete a variable from the various namespaces, so that, as
1441 """Delete a variable from the various namespaces, so that, as
1442 far as possible, we're not keeping any hidden references to it.
1442 far as possible, we're not keeping any hidden references to it.
1443
1443
1444 Parameters
1444 Parameters
1445 ----------
1445 ----------
1446 varname : str
1446 varname : str
1447 The name of the variable to delete.
1447 The name of the variable to delete.
1448 by_name : bool
1448 by_name : bool
1449 If True, delete variables with the given name in each
1449 If True, delete variables with the given name in each
1450 namespace. If False (default), find the variable in the user
1450 namespace. If False (default), find the variable in the user
1451 namespace, and delete references to it.
1451 namespace, and delete references to it.
1452 """
1452 """
1453 if varname in ('__builtin__', '__builtins__'):
1453 if varname in ('__builtin__', '__builtins__'):
1454 raise ValueError("Refusing to delete %s" % varname)
1454 raise ValueError("Refusing to delete %s" % varname)
1455
1455
1456 ns_refs = self.all_ns_refs
1456 ns_refs = self.all_ns_refs
1457
1457
1458 if by_name: # Delete by name
1458 if by_name: # Delete by name
1459 for ns in ns_refs:
1459 for ns in ns_refs:
1460 try:
1460 try:
1461 del ns[varname]
1461 del ns[varname]
1462 except KeyError:
1462 except KeyError:
1463 pass
1463 pass
1464 else: # Delete by object
1464 else: # Delete by object
1465 try:
1465 try:
1466 obj = self.user_ns[varname]
1466 obj = self.user_ns[varname]
1467 except KeyError:
1467 except KeyError:
1468 raise NameError("name '%s' is not defined" % varname)
1468 raise NameError("name '%s' is not defined" % varname)
1469 # Also check in output history
1469 # Also check in output history
1470 ns_refs.append(self.history_manager.output_hist)
1470 ns_refs.append(self.history_manager.output_hist)
1471 for ns in ns_refs:
1471 for ns in ns_refs:
1472 to_delete = [n for n, o in ns.items() if o is obj]
1472 to_delete = [n for n, o in ns.items() if o is obj]
1473 for name in to_delete:
1473 for name in to_delete:
1474 del ns[name]
1474 del ns[name]
1475
1475
1476 # Ensure it is removed from the last execution result
1476 # Ensure it is removed from the last execution result
1477 if self.last_execution_result.result is obj:
1477 if self.last_execution_result.result is obj:
1478 self.last_execution_result = None
1478 self.last_execution_result = None
1479
1479
1480 # displayhook keeps extra references, but not in a dictionary
1480 # displayhook keeps extra references, but not in a dictionary
1481 for name in ('_', '__', '___'):
1481 for name in ('_', '__', '___'):
1482 if getattr(self.displayhook, name) is obj:
1482 if getattr(self.displayhook, name) is obj:
1483 setattr(self.displayhook, name, None)
1483 setattr(self.displayhook, name, None)
1484
1484
1485 def reset_selective(self, regex=None):
1485 def reset_selective(self, regex=None):
1486 """Clear selective variables from internal namespaces based on a
1486 """Clear selective variables from internal namespaces based on a
1487 specified regular expression.
1487 specified regular expression.
1488
1488
1489 Parameters
1489 Parameters
1490 ----------
1490 ----------
1491 regex : string or compiled pattern, optional
1491 regex : string or compiled pattern, optional
1492 A regular expression pattern that will be used in searching
1492 A regular expression pattern that will be used in searching
1493 variable names in the users namespaces.
1493 variable names in the users namespaces.
1494 """
1494 """
1495 if regex is not None:
1495 if regex is not None:
1496 try:
1496 try:
1497 m = re.compile(regex)
1497 m = re.compile(regex)
1498 except TypeError:
1498 except TypeError:
1499 raise TypeError('regex must be a string or compiled pattern')
1499 raise TypeError('regex must be a string or compiled pattern')
1500 # Search for keys in each namespace that match the given regex
1500 # Search for keys in each namespace that match the given regex
1501 # If a match is found, delete the key/value pair.
1501 # If a match is found, delete the key/value pair.
1502 for ns in self.all_ns_refs:
1502 for ns in self.all_ns_refs:
1503 for var in ns:
1503 for var in ns:
1504 if m.search(var):
1504 if m.search(var):
1505 del ns[var]
1505 del ns[var]
1506
1506
1507 def push(self, variables, interactive=True):
1507 def push(self, variables, interactive=True):
1508 """Inject a group of variables into the IPython user namespace.
1508 """Inject a group of variables into the IPython user namespace.
1509
1509
1510 Parameters
1510 Parameters
1511 ----------
1511 ----------
1512 variables : dict, str or list/tuple of str
1512 variables : dict, str or list/tuple of str
1513 The variables to inject into the user's namespace. If a dict, a
1513 The variables to inject into the user's namespace. If a dict, a
1514 simple update is done. If a str, the string is assumed to have
1514 simple update is done. If a str, the string is assumed to have
1515 variable names separated by spaces. A list/tuple of str can also
1515 variable names separated by spaces. A list/tuple of str can also
1516 be used to give the variable names. If just the variable names are
1516 be used to give the variable names. If just the variable names are
1517 give (list/tuple/str) then the variable values looked up in the
1517 give (list/tuple/str) then the variable values looked up in the
1518 callers frame.
1518 callers frame.
1519 interactive : bool
1519 interactive : bool
1520 If True (default), the variables will be listed with the ``who``
1520 If True (default), the variables will be listed with the ``who``
1521 magic.
1521 magic.
1522 """
1522 """
1523 vdict = None
1523 vdict = None
1524
1524
1525 # We need a dict of name/value pairs to do namespace updates.
1525 # We need a dict of name/value pairs to do namespace updates.
1526 if isinstance(variables, dict):
1526 if isinstance(variables, dict):
1527 vdict = variables
1527 vdict = variables
1528 elif isinstance(variables, (str, list, tuple)):
1528 elif isinstance(variables, (str, list, tuple)):
1529 if isinstance(variables, str):
1529 if isinstance(variables, str):
1530 vlist = variables.split()
1530 vlist = variables.split()
1531 else:
1531 else:
1532 vlist = variables
1532 vlist = variables
1533 vdict = {}
1533 vdict = {}
1534 cf = sys._getframe(1)
1534 cf = sys._getframe(1)
1535 for name in vlist:
1535 for name in vlist:
1536 try:
1536 try:
1537 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1537 vdict[name] = eval(name, cf.f_globals, cf.f_locals)
1538 except:
1538 except:
1539 print('Could not get variable %s from %s' %
1539 print('Could not get variable %s from %s' %
1540 (name,cf.f_code.co_name))
1540 (name,cf.f_code.co_name))
1541 else:
1541 else:
1542 raise ValueError('variables must be a dict/str/list/tuple')
1542 raise ValueError('variables must be a dict/str/list/tuple')
1543
1543
1544 # Propagate variables to user namespace
1544 # Propagate variables to user namespace
1545 self.user_ns.update(vdict)
1545 self.user_ns.update(vdict)
1546
1546
1547 # And configure interactive visibility
1547 # And configure interactive visibility
1548 user_ns_hidden = self.user_ns_hidden
1548 user_ns_hidden = self.user_ns_hidden
1549 if interactive:
1549 if interactive:
1550 for name in vdict:
1550 for name in vdict:
1551 user_ns_hidden.pop(name, None)
1551 user_ns_hidden.pop(name, None)
1552 else:
1552 else:
1553 user_ns_hidden.update(vdict)
1553 user_ns_hidden.update(vdict)
1554
1554
1555 def drop_by_id(self, variables):
1555 def drop_by_id(self, variables):
1556 """Remove a dict of variables from the user namespace, if they are the
1556 """Remove a dict of variables from the user namespace, if they are the
1557 same as the values in the dictionary.
1557 same as the values in the dictionary.
1558
1558
1559 This is intended for use by extensions: variables that they've added can
1559 This is intended for use by extensions: variables that they've added can
1560 be taken back out if they are unloaded, without removing any that the
1560 be taken back out if they are unloaded, without removing any that the
1561 user has overwritten.
1561 user has overwritten.
1562
1562
1563 Parameters
1563 Parameters
1564 ----------
1564 ----------
1565 variables : dict
1565 variables : dict
1566 A dictionary mapping object names (as strings) to the objects.
1566 A dictionary mapping object names (as strings) to the objects.
1567 """
1567 """
1568 for name, obj in variables.items():
1568 for name, obj in variables.items():
1569 if name in self.user_ns and self.user_ns[name] is obj:
1569 if name in self.user_ns and self.user_ns[name] is obj:
1570 del self.user_ns[name]
1570 del self.user_ns[name]
1571 self.user_ns_hidden.pop(name, None)
1571 self.user_ns_hidden.pop(name, None)
1572
1572
1573 #-------------------------------------------------------------------------
1573 #-------------------------------------------------------------------------
1574 # Things related to object introspection
1574 # Things related to object introspection
1575 #-------------------------------------------------------------------------
1575 #-------------------------------------------------------------------------
1576
1576
1577 def _ofind(self, oname, namespaces=None):
1577 def _ofind(self, oname, namespaces=None):
1578 """Find an object in the available namespaces.
1578 """Find an object in the available namespaces.
1579
1579
1580 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1580 self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
1581
1581
1582 Has special code to detect magic functions.
1582 Has special code to detect magic functions.
1583 """
1583 """
1584 oname = oname.strip()
1584 oname = oname.strip()
1585 if not oname.startswith(ESC_MAGIC) and \
1585 if not oname.startswith(ESC_MAGIC) and \
1586 not oname.startswith(ESC_MAGIC2) and \
1586 not oname.startswith(ESC_MAGIC2) and \
1587 not all(a.isidentifier() for a in oname.split(".")):
1587 not all(a.isidentifier() for a in oname.split(".")):
1588 return {'found': False}
1588 return {'found': False}
1589
1589
1590 if namespaces is None:
1590 if namespaces is None:
1591 # Namespaces to search in:
1591 # Namespaces to search in:
1592 # Put them in a list. The order is important so that we
1592 # Put them in a list. The order is important so that we
1593 # find things in the same order that Python finds them.
1593 # find things in the same order that Python finds them.
1594 namespaces = [ ('Interactive', self.user_ns),
1594 namespaces = [ ('Interactive', self.user_ns),
1595 ('Interactive (global)', self.user_global_ns),
1595 ('Interactive (global)', self.user_global_ns),
1596 ('Python builtin', builtin_mod.__dict__),
1596 ('Python builtin', builtin_mod.__dict__),
1597 ]
1597 ]
1598
1598
1599 ismagic = False
1599 ismagic = False
1600 isalias = False
1600 isalias = False
1601 found = False
1601 found = False
1602 ospace = None
1602 ospace = None
1603 parent = None
1603 parent = None
1604 obj = None
1604 obj = None
1605
1605
1606
1606
1607 # Look for the given name by splitting it in parts. If the head is
1607 # Look for the given name by splitting it in parts. If the head is
1608 # found, then we look for all the remaining parts as members, and only
1608 # found, then we look for all the remaining parts as members, and only
1609 # declare success if we can find them all.
1609 # declare success if we can find them all.
1610 oname_parts = oname.split('.')
1610 oname_parts = oname.split('.')
1611 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1611 oname_head, oname_rest = oname_parts[0],oname_parts[1:]
1612 for nsname,ns in namespaces:
1612 for nsname,ns in namespaces:
1613 try:
1613 try:
1614 obj = ns[oname_head]
1614 obj = ns[oname_head]
1615 except KeyError:
1615 except KeyError:
1616 continue
1616 continue
1617 else:
1617 else:
1618 for idx, part in enumerate(oname_rest):
1618 for idx, part in enumerate(oname_rest):
1619 try:
1619 try:
1620 parent = obj
1620 parent = obj
1621 # The last part is looked up in a special way to avoid
1621 # The last part is looked up in a special way to avoid
1622 # descriptor invocation as it may raise or have side
1622 # descriptor invocation as it may raise or have side
1623 # effects.
1623 # effects.
1624 if idx == len(oname_rest) - 1:
1624 if idx == len(oname_rest) - 1:
1625 obj = self._getattr_property(obj, part)
1625 obj = self._getattr_property(obj, part)
1626 else:
1626 else:
1627 obj = getattr(obj, part)
1627 obj = getattr(obj, part)
1628 except:
1628 except:
1629 # Blanket except b/c some badly implemented objects
1629 # Blanket except b/c some badly implemented objects
1630 # allow __getattr__ to raise exceptions other than
1630 # allow __getattr__ to raise exceptions other than
1631 # AttributeError, which then crashes IPython.
1631 # AttributeError, which then crashes IPython.
1632 break
1632 break
1633 else:
1633 else:
1634 # If we finish the for loop (no break), we got all members
1634 # If we finish the for loop (no break), we got all members
1635 found = True
1635 found = True
1636 ospace = nsname
1636 ospace = nsname
1637 break # namespace loop
1637 break # namespace loop
1638
1638
1639 # Try to see if it's magic
1639 # Try to see if it's magic
1640 if not found:
1640 if not found:
1641 obj = None
1641 obj = None
1642 if oname.startswith(ESC_MAGIC2):
1642 if oname.startswith(ESC_MAGIC2):
1643 oname = oname.lstrip(ESC_MAGIC2)
1643 oname = oname.lstrip(ESC_MAGIC2)
1644 obj = self.find_cell_magic(oname)
1644 obj = self.find_cell_magic(oname)
1645 elif oname.startswith(ESC_MAGIC):
1645 elif oname.startswith(ESC_MAGIC):
1646 oname = oname.lstrip(ESC_MAGIC)
1646 oname = oname.lstrip(ESC_MAGIC)
1647 obj = self.find_line_magic(oname)
1647 obj = self.find_line_magic(oname)
1648 else:
1648 else:
1649 # search without prefix, so run? will find %run?
1649 # search without prefix, so run? will find %run?
1650 obj = self.find_line_magic(oname)
1650 obj = self.find_line_magic(oname)
1651 if obj is None:
1651 if obj is None:
1652 obj = self.find_cell_magic(oname)
1652 obj = self.find_cell_magic(oname)
1653 if obj is not None:
1653 if obj is not None:
1654 found = True
1654 found = True
1655 ospace = 'IPython internal'
1655 ospace = 'IPython internal'
1656 ismagic = True
1656 ismagic = True
1657 isalias = isinstance(obj, Alias)
1657 isalias = isinstance(obj, Alias)
1658
1658
1659 # Last try: special-case some literals like '', [], {}, etc:
1659 # Last try: special-case some literals like '', [], {}, etc:
1660 if not found and oname_head in ["''",'""','[]','{}','()']:
1660 if not found and oname_head in ["''",'""','[]','{}','()']:
1661 obj = eval(oname_head)
1661 obj = eval(oname_head)
1662 found = True
1662 found = True
1663 ospace = 'Interactive'
1663 ospace = 'Interactive'
1664
1664
1665 return {
1665 return {
1666 'obj':obj,
1666 'obj':obj,
1667 'found':found,
1667 'found':found,
1668 'parent':parent,
1668 'parent':parent,
1669 'ismagic':ismagic,
1669 'ismagic':ismagic,
1670 'isalias':isalias,
1670 'isalias':isalias,
1671 'namespace':ospace
1671 'namespace':ospace
1672 }
1672 }
1673
1673
1674 @staticmethod
1674 @staticmethod
1675 def _getattr_property(obj, attrname):
1675 def _getattr_property(obj, attrname):
1676 """Property-aware getattr to use in object finding.
1676 """Property-aware getattr to use in object finding.
1677
1677
1678 If attrname represents a property, return it unevaluated (in case it has
1678 If attrname represents a property, return it unevaluated (in case it has
1679 side effects or raises an error.
1679 side effects or raises an error.
1680
1680
1681 """
1681 """
1682 if not isinstance(obj, type):
1682 if not isinstance(obj, type):
1683 try:
1683 try:
1684 # `getattr(type(obj), attrname)` is not guaranteed to return
1684 # `getattr(type(obj), attrname)` is not guaranteed to return
1685 # `obj`, but does so for property:
1685 # `obj`, but does so for property:
1686 #
1686 #
1687 # property.__get__(self, None, cls) -> self
1687 # property.__get__(self, None, cls) -> self
1688 #
1688 #
1689 # The universal alternative is to traverse the mro manually
1689 # The universal alternative is to traverse the mro manually
1690 # searching for attrname in class dicts.
1690 # searching for attrname in class dicts.
1691 attr = getattr(type(obj), attrname)
1691 attr = getattr(type(obj), attrname)
1692 except AttributeError:
1692 except AttributeError:
1693 pass
1693 pass
1694 else:
1694 else:
1695 # This relies on the fact that data descriptors (with both
1695 # This relies on the fact that data descriptors (with both
1696 # __get__ & __set__ magic methods) take precedence over
1696 # __get__ & __set__ magic methods) take precedence over
1697 # instance-level attributes:
1697 # instance-level attributes:
1698 #
1698 #
1699 # class A(object):
1699 # class A(object):
1700 # @property
1700 # @property
1701 # def foobar(self): return 123
1701 # def foobar(self): return 123
1702 # a = A()
1702 # a = A()
1703 # a.__dict__['foobar'] = 345
1703 # a.__dict__['foobar'] = 345
1704 # a.foobar # == 123
1704 # a.foobar # == 123
1705 #
1705 #
1706 # So, a property may be returned right away.
1706 # So, a property may be returned right away.
1707 if isinstance(attr, property):
1707 if isinstance(attr, property):
1708 return attr
1708 return attr
1709
1709
1710 # Nothing helped, fall back.
1710 # Nothing helped, fall back.
1711 return getattr(obj, attrname)
1711 return getattr(obj, attrname)
1712
1712
1713 def _object_find(self, oname, namespaces=None):
1713 def _object_find(self, oname, namespaces=None):
1714 """Find an object and return a struct with info about it."""
1714 """Find an object and return a struct with info about it."""
1715 return Struct(self._ofind(oname, namespaces))
1715 return Struct(self._ofind(oname, namespaces))
1716
1716
1717 def _inspect(self, meth, oname, namespaces=None, **kw):
1717 def _inspect(self, meth, oname, namespaces=None, **kw):
1718 """Generic interface to the inspector system.
1718 """Generic interface to the inspector system.
1719
1719
1720 This function is meant to be called by pdef, pdoc & friends.
1720 This function is meant to be called by pdef, pdoc & friends.
1721 """
1721 """
1722 info = self._object_find(oname, namespaces)
1722 info = self._object_find(oname, namespaces)
1723 docformat = sphinxify if self.sphinxify_docstring else None
1723 docformat = sphinxify if self.sphinxify_docstring else None
1724 if info.found:
1724 if info.found:
1725 pmethod = getattr(self.inspector, meth)
1725 pmethod = getattr(self.inspector, meth)
1726 # TODO: only apply format_screen to the plain/text repr of the mime
1726 # TODO: only apply format_screen to the plain/text repr of the mime
1727 # bundle.
1727 # bundle.
1728 formatter = format_screen if info.ismagic else docformat
1728 formatter = format_screen if info.ismagic else docformat
1729 if meth == 'pdoc':
1729 if meth == 'pdoc':
1730 pmethod(info.obj, oname, formatter)
1730 pmethod(info.obj, oname, formatter)
1731 elif meth == 'pinfo':
1731 elif meth == 'pinfo':
1732 pmethod(info.obj, oname, formatter, info,
1732 pmethod(info.obj, oname, formatter, info,
1733 enable_html_pager=self.enable_html_pager, **kw)
1733 enable_html_pager=self.enable_html_pager, **kw)
1734 else:
1734 else:
1735 pmethod(info.obj, oname)
1735 pmethod(info.obj, oname)
1736 else:
1736 else:
1737 print('Object `%s` not found.' % oname)
1737 print('Object `%s` not found.' % oname)
1738 return 'not found' # so callers can take other action
1738 return 'not found' # so callers can take other action
1739
1739
1740 def object_inspect(self, oname, detail_level=0):
1740 def object_inspect(self, oname, detail_level=0):
1741 """Get object info about oname"""
1741 """Get object info about oname"""
1742 with self.builtin_trap:
1742 with self.builtin_trap:
1743 info = self._object_find(oname)
1743 info = self._object_find(oname)
1744 if info.found:
1744 if info.found:
1745 return self.inspector.info(info.obj, oname, info=info,
1745 return self.inspector.info(info.obj, oname, info=info,
1746 detail_level=detail_level
1746 detail_level=detail_level
1747 )
1747 )
1748 else:
1748 else:
1749 return oinspect.object_info(name=oname, found=False)
1749 return oinspect.object_info(name=oname, found=False)
1750
1750
1751 def object_inspect_text(self, oname, detail_level=0):
1751 def object_inspect_text(self, oname, detail_level=0):
1752 """Get object info as formatted text"""
1752 """Get object info as formatted text"""
1753 return self.object_inspect_mime(oname, detail_level)['text/plain']
1753 return self.object_inspect_mime(oname, detail_level)['text/plain']
1754
1754
1755 def object_inspect_mime(self, oname, detail_level=0):
1755 def object_inspect_mime(self, oname, detail_level=0):
1756 """Get object info as a mimebundle of formatted representations.
1756 """Get object info as a mimebundle of formatted representations.
1757
1757
1758 A mimebundle is a dictionary, keyed by mime-type.
1758 A mimebundle is a dictionary, keyed by mime-type.
1759 It must always have the key `'text/plain'`.
1759 It must always have the key `'text/plain'`.
1760 """
1760 """
1761 with self.builtin_trap:
1761 with self.builtin_trap:
1762 info = self._object_find(oname)
1762 info = self._object_find(oname)
1763 if info.found:
1763 if info.found:
1764 return self.inspector._get_info(info.obj, oname, info=info,
1764 return self.inspector._get_info(info.obj, oname, info=info,
1765 detail_level=detail_level
1765 detail_level=detail_level
1766 )
1766 )
1767 else:
1767 else:
1768 raise KeyError(oname)
1768 raise KeyError(oname)
1769
1769
1770 #-------------------------------------------------------------------------
1770 #-------------------------------------------------------------------------
1771 # Things related to history management
1771 # Things related to history management
1772 #-------------------------------------------------------------------------
1772 #-------------------------------------------------------------------------
1773
1773
1774 def init_history(self):
1774 def init_history(self):
1775 """Sets up the command history, and starts regular autosaves."""
1775 """Sets up the command history, and starts regular autosaves."""
1776 self.history_manager = HistoryManager(shell=self, parent=self)
1776 self.history_manager = HistoryManager(shell=self, parent=self)
1777 self.configurables.append(self.history_manager)
1777 self.configurables.append(self.history_manager)
1778
1778
1779 #-------------------------------------------------------------------------
1779 #-------------------------------------------------------------------------
1780 # Things related to exception handling and tracebacks (not debugging)
1780 # Things related to exception handling and tracebacks (not debugging)
1781 #-------------------------------------------------------------------------
1781 #-------------------------------------------------------------------------
1782
1782
1783 debugger_cls = Pdb
1783 debugger_cls = Pdb
1784
1784
1785 def init_traceback_handlers(self, custom_exceptions):
1785 def init_traceback_handlers(self, custom_exceptions):
1786 # Syntax error handler.
1786 # Syntax error handler.
1787 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
1787 self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
1788
1788
1789 # The interactive one is initialized with an offset, meaning we always
1789 # The interactive one is initialized with an offset, meaning we always
1790 # want to remove the topmost item in the traceback, which is our own
1790 # want to remove the topmost item in the traceback, which is our own
1791 # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
1791 # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
1792 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1792 self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
1793 color_scheme='NoColor',
1793 color_scheme='NoColor',
1794 tb_offset = 1,
1794 tb_offset = 1,
1795 check_cache=check_linecache_ipython,
1795 check_cache=check_linecache_ipython,
1796 debugger_cls=self.debugger_cls, parent=self)
1796 debugger_cls=self.debugger_cls, parent=self)
1797
1797
1798 # The instance will store a pointer to the system-wide exception hook,
1798 # The instance will store a pointer to the system-wide exception hook,
1799 # so that runtime code (such as magics) can access it. This is because
1799 # so that runtime code (such as magics) can access it. This is because
1800 # during the read-eval loop, it may get temporarily overwritten.
1800 # during the read-eval loop, it may get temporarily overwritten.
1801 self.sys_excepthook = sys.excepthook
1801 self.sys_excepthook = sys.excepthook
1802
1802
1803 # and add any custom exception handlers the user may have specified
1803 # and add any custom exception handlers the user may have specified
1804 self.set_custom_exc(*custom_exceptions)
1804 self.set_custom_exc(*custom_exceptions)
1805
1805
1806 # Set the exception mode
1806 # Set the exception mode
1807 self.InteractiveTB.set_mode(mode=self.xmode)
1807 self.InteractiveTB.set_mode(mode=self.xmode)
1808
1808
1809 def set_custom_exc(self, exc_tuple, handler):
1809 def set_custom_exc(self, exc_tuple, handler):
1810 """set_custom_exc(exc_tuple, handler)
1810 """set_custom_exc(exc_tuple, handler)
1811
1811
1812 Set a custom exception handler, which will be called if any of the
1812 Set a custom exception handler, which will be called if any of the
1813 exceptions in exc_tuple occur in the mainloop (specifically, in the
1813 exceptions in exc_tuple occur in the mainloop (specifically, in the
1814 run_code() method).
1814 run_code() method).
1815
1815
1816 Parameters
1816 Parameters
1817 ----------
1817 ----------
1818
1818
1819 exc_tuple : tuple of exception classes
1819 exc_tuple : tuple of exception classes
1820 A *tuple* of exception classes, for which to call the defined
1820 A *tuple* of exception classes, for which to call the defined
1821 handler. It is very important that you use a tuple, and NOT A
1821 handler. It is very important that you use a tuple, and NOT A
1822 LIST here, because of the way Python's except statement works. If
1822 LIST here, because of the way Python's except statement works. If
1823 you only want to trap a single exception, use a singleton tuple::
1823 you only want to trap a single exception, use a singleton tuple::
1824
1824
1825 exc_tuple == (MyCustomException,)
1825 exc_tuple == (MyCustomException,)
1826
1826
1827 handler : callable
1827 handler : callable
1828 handler must have the following signature::
1828 handler must have the following signature::
1829
1829
1830 def my_handler(self, etype, value, tb, tb_offset=None):
1830 def my_handler(self, etype, value, tb, tb_offset=None):
1831 ...
1831 ...
1832 return structured_traceback
1832 return structured_traceback
1833
1833
1834 Your handler must return a structured traceback (a list of strings),
1834 Your handler must return a structured traceback (a list of strings),
1835 or None.
1835 or None.
1836
1836
1837 This will be made into an instance method (via types.MethodType)
1837 This will be made into an instance method (via types.MethodType)
1838 of IPython itself, and it will be called if any of the exceptions
1838 of IPython itself, and it will be called if any of the exceptions
1839 listed in the exc_tuple are caught. If the handler is None, an
1839 listed in the exc_tuple are caught. If the handler is None, an
1840 internal basic one is used, which just prints basic info.
1840 internal basic one is used, which just prints basic info.
1841
1841
1842 To protect IPython from crashes, if your handler ever raises an
1842 To protect IPython from crashes, if your handler ever raises an
1843 exception or returns an invalid result, it will be immediately
1843 exception or returns an invalid result, it will be immediately
1844 disabled.
1844 disabled.
1845
1845
1846 WARNING: by putting in your own exception handler into IPython's main
1846 WARNING: by putting in your own exception handler into IPython's main
1847 execution loop, you run a very good chance of nasty crashes. This
1847 execution loop, you run a very good chance of nasty crashes. This
1848 facility should only be used if you really know what you are doing."""
1848 facility should only be used if you really know what you are doing."""
1849 if not isinstance(exc_tuple, tuple):
1849 if not isinstance(exc_tuple, tuple):
1850 raise TypeError("The custom exceptions must be given as a tuple.")
1850 raise TypeError("The custom exceptions must be given as a tuple.")
1851
1851
1852 def dummy_handler(self, etype, value, tb, tb_offset=None):
1852 def dummy_handler(self, etype, value, tb, tb_offset=None):
1853 print('*** Simple custom exception handler ***')
1853 print('*** Simple custom exception handler ***')
1854 print('Exception type :', etype)
1854 print('Exception type :', etype)
1855 print('Exception value:', value)
1855 print('Exception value:', value)
1856 print('Traceback :', tb)
1856 print('Traceback :', tb)
1857
1857
1858 def validate_stb(stb):
1858 def validate_stb(stb):
1859 """validate structured traceback return type
1859 """validate structured traceback return type
1860
1860
1861 return type of CustomTB *should* be a list of strings, but allow
1861 return type of CustomTB *should* be a list of strings, but allow
1862 single strings or None, which are harmless.
1862 single strings or None, which are harmless.
1863
1863
1864 This function will *always* return a list of strings,
1864 This function will *always* return a list of strings,
1865 and will raise a TypeError if stb is inappropriate.
1865 and will raise a TypeError if stb is inappropriate.
1866 """
1866 """
1867 msg = "CustomTB must return list of strings, not %r" % stb
1867 msg = "CustomTB must return list of strings, not %r" % stb
1868 if stb is None:
1868 if stb is None:
1869 return []
1869 return []
1870 elif isinstance(stb, str):
1870 elif isinstance(stb, str):
1871 return [stb]
1871 return [stb]
1872 elif not isinstance(stb, list):
1872 elif not isinstance(stb, list):
1873 raise TypeError(msg)
1873 raise TypeError(msg)
1874 # it's a list
1874 # it's a list
1875 for line in stb:
1875 for line in stb:
1876 # check every element
1876 # check every element
1877 if not isinstance(line, str):
1877 if not isinstance(line, str):
1878 raise TypeError(msg)
1878 raise TypeError(msg)
1879 return stb
1879 return stb
1880
1880
1881 if handler is None:
1881 if handler is None:
1882 wrapped = dummy_handler
1882 wrapped = dummy_handler
1883 else:
1883 else:
1884 def wrapped(self,etype,value,tb,tb_offset=None):
1884 def wrapped(self,etype,value,tb,tb_offset=None):
1885 """wrap CustomTB handler, to protect IPython from user code
1885 """wrap CustomTB handler, to protect IPython from user code
1886
1886
1887 This makes it harder (but not impossible) for custom exception
1887 This makes it harder (but not impossible) for custom exception
1888 handlers to crash IPython.
1888 handlers to crash IPython.
1889 """
1889 """
1890 try:
1890 try:
1891 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1891 stb = handler(self,etype,value,tb,tb_offset=tb_offset)
1892 return validate_stb(stb)
1892 return validate_stb(stb)
1893 except:
1893 except:
1894 # clear custom handler immediately
1894 # clear custom handler immediately
1895 self.set_custom_exc((), None)
1895 self.set_custom_exc((), None)
1896 print("Custom TB Handler failed, unregistering", file=sys.stderr)
1896 print("Custom TB Handler failed, unregistering", file=sys.stderr)
1897 # show the exception in handler first
1897 # show the exception in handler first
1898 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1898 stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
1899 print(self.InteractiveTB.stb2text(stb))
1899 print(self.InteractiveTB.stb2text(stb))
1900 print("The original exception:")
1900 print("The original exception:")
1901 stb = self.InteractiveTB.structured_traceback(
1901 stb = self.InteractiveTB.structured_traceback(
1902 (etype,value,tb), tb_offset=tb_offset
1902 (etype,value,tb), tb_offset=tb_offset
1903 )
1903 )
1904 return stb
1904 return stb
1905
1905
1906 self.CustomTB = types.MethodType(wrapped,self)
1906 self.CustomTB = types.MethodType(wrapped,self)
1907 self.custom_exceptions = exc_tuple
1907 self.custom_exceptions = exc_tuple
1908
1908
1909 def excepthook(self, etype, value, tb):
1909 def excepthook(self, etype, value, tb):
1910 """One more defense for GUI apps that call sys.excepthook.
1910 """One more defense for GUI apps that call sys.excepthook.
1911
1911
1912 GUI frameworks like wxPython trap exceptions and call
1912 GUI frameworks like wxPython trap exceptions and call
1913 sys.excepthook themselves. I guess this is a feature that
1913 sys.excepthook themselves. I guess this is a feature that
1914 enables them to keep running after exceptions that would
1914 enables them to keep running after exceptions that would
1915 otherwise kill their mainloop. This is a bother for IPython
1915 otherwise kill their mainloop. This is a bother for IPython
1916 which excepts to catch all of the program exceptions with a try:
1916 which excepts to catch all of the program exceptions with a try:
1917 except: statement.
1917 except: statement.
1918
1918
1919 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1919 Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
1920 any app directly invokes sys.excepthook, it will look to the user like
1920 any app directly invokes sys.excepthook, it will look to the user like
1921 IPython crashed. In order to work around this, we can disable the
1921 IPython crashed. In order to work around this, we can disable the
1922 CrashHandler and replace it with this excepthook instead, which prints a
1922 CrashHandler and replace it with this excepthook instead, which prints a
1923 regular traceback using our InteractiveTB. In this fashion, apps which
1923 regular traceback using our InteractiveTB. In this fashion, apps which
1924 call sys.excepthook will generate a regular-looking exception from
1924 call sys.excepthook will generate a regular-looking exception from
1925 IPython, and the CrashHandler will only be triggered by real IPython
1925 IPython, and the CrashHandler will only be triggered by real IPython
1926 crashes.
1926 crashes.
1927
1927
1928 This hook should be used sparingly, only in places which are not likely
1928 This hook should be used sparingly, only in places which are not likely
1929 to be true IPython errors.
1929 to be true IPython errors.
1930 """
1930 """
1931 self.showtraceback((etype, value, tb), tb_offset=0)
1931 self.showtraceback((etype, value, tb), tb_offset=0)
1932
1932
1933 def _get_exc_info(self, exc_tuple=None):
1933 def _get_exc_info(self, exc_tuple=None):
1934 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1934 """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
1935
1935
1936 Ensures sys.last_type,value,traceback hold the exc_info we found,
1936 Ensures sys.last_type,value,traceback hold the exc_info we found,
1937 from whichever source.
1937 from whichever source.
1938
1938
1939 raises ValueError if none of these contain any information
1939 raises ValueError if none of these contain any information
1940 """
1940 """
1941 if exc_tuple is None:
1941 if exc_tuple is None:
1942 etype, value, tb = sys.exc_info()
1942 etype, value, tb = sys.exc_info()
1943 else:
1943 else:
1944 etype, value, tb = exc_tuple
1944 etype, value, tb = exc_tuple
1945
1945
1946 if etype is None:
1946 if etype is None:
1947 if hasattr(sys, 'last_type'):
1947 if hasattr(sys, 'last_type'):
1948 etype, value, tb = sys.last_type, sys.last_value, \
1948 etype, value, tb = sys.last_type, sys.last_value, \
1949 sys.last_traceback
1949 sys.last_traceback
1950
1950
1951 if etype is None:
1951 if etype is None:
1952 raise ValueError("No exception to find")
1952 raise ValueError("No exception to find")
1953
1953
1954 # Now store the exception info in sys.last_type etc.
1954 # Now store the exception info in sys.last_type etc.
1955 # WARNING: these variables are somewhat deprecated and not
1955 # WARNING: these variables are somewhat deprecated and not
1956 # necessarily safe to use in a threaded environment, but tools
1956 # necessarily safe to use in a threaded environment, but tools
1957 # like pdb depend on their existence, so let's set them. If we
1957 # like pdb depend on their existence, so let's set them. If we
1958 # find problems in the field, we'll need to revisit their use.
1958 # find problems in the field, we'll need to revisit their use.
1959 sys.last_type = etype
1959 sys.last_type = etype
1960 sys.last_value = value
1960 sys.last_value = value
1961 sys.last_traceback = tb
1961 sys.last_traceback = tb
1962
1962
1963 return etype, value, tb
1963 return etype, value, tb
1964
1964
1965 def show_usage_error(self, exc):
1965 def show_usage_error(self, exc):
1966 """Show a short message for UsageErrors
1966 """Show a short message for UsageErrors
1967
1967
1968 These are special exceptions that shouldn't show a traceback.
1968 These are special exceptions that shouldn't show a traceback.
1969 """
1969 """
1970 print("UsageError: %s" % exc, file=sys.stderr)
1970 print("UsageError: %s" % exc, file=sys.stderr)
1971
1971
1972 def get_exception_only(self, exc_tuple=None):
1972 def get_exception_only(self, exc_tuple=None):
1973 """
1973 """
1974 Return as a string (ending with a newline) the exception that
1974 Return as a string (ending with a newline) the exception that
1975 just occurred, without any traceback.
1975 just occurred, without any traceback.
1976 """
1976 """
1977 etype, value, tb = self._get_exc_info(exc_tuple)
1977 etype, value, tb = self._get_exc_info(exc_tuple)
1978 msg = traceback.format_exception_only(etype, value)
1978 msg = traceback.format_exception_only(etype, value)
1979 return ''.join(msg)
1979 return ''.join(msg)
1980
1980
1981 def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
1981 def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
1982 exception_only=False, running_compiled_code=False):
1982 exception_only=False, running_compiled_code=False):
1983 """Display the exception that just occurred.
1983 """Display the exception that just occurred.
1984
1984
1985 If nothing is known about the exception, this is the method which
1985 If nothing is known about the exception, this is the method which
1986 should be used throughout the code for presenting user tracebacks,
1986 should be used throughout the code for presenting user tracebacks,
1987 rather than directly invoking the InteractiveTB object.
1987 rather than directly invoking the InteractiveTB object.
1988
1988
1989 A specific showsyntaxerror() also exists, but this method can take
1989 A specific showsyntaxerror() also exists, but this method can take
1990 care of calling it if needed, so unless you are explicitly catching a
1990 care of calling it if needed, so unless you are explicitly catching a
1991 SyntaxError exception, don't try to analyze the stack manually and
1991 SyntaxError exception, don't try to analyze the stack manually and
1992 simply call this method."""
1992 simply call this method."""
1993
1993
1994 try:
1994 try:
1995 try:
1995 try:
1996 etype, value, tb = self._get_exc_info(exc_tuple)
1996 etype, value, tb = self._get_exc_info(exc_tuple)
1997 except ValueError:
1997 except ValueError:
1998 print('No traceback available to show.', file=sys.stderr)
1998 print('No traceback available to show.', file=sys.stderr)
1999 return
1999 return
2000
2000
2001 if issubclass(etype, SyntaxError):
2001 if issubclass(etype, SyntaxError):
2002 # Though this won't be called by syntax errors in the input
2002 # Though this won't be called by syntax errors in the input
2003 # line, there may be SyntaxError cases with imported code.
2003 # line, there may be SyntaxError cases with imported code.
2004 self.showsyntaxerror(filename, running_compiled_code)
2004 self.showsyntaxerror(filename, running_compiled_code)
2005 elif etype is UsageError:
2005 elif etype is UsageError:
2006 self.show_usage_error(value)
2006 self.show_usage_error(value)
2007 else:
2007 else:
2008 if exception_only:
2008 if exception_only:
2009 stb = ['An exception has occurred, use %tb to see '
2009 stb = ['An exception has occurred, use %tb to see '
2010 'the full traceback.\n']
2010 'the full traceback.\n']
2011 stb.extend(self.InteractiveTB.get_exception_only(etype,
2011 stb.extend(self.InteractiveTB.get_exception_only(etype,
2012 value))
2012 value))
2013 else:
2013 else:
2014 try:
2014 try:
2015 # Exception classes can customise their traceback - we
2015 # Exception classes can customise their traceback - we
2016 # use this in IPython.parallel for exceptions occurring
2016 # use this in IPython.parallel for exceptions occurring
2017 # in the engines. This should return a list of strings.
2017 # in the engines. This should return a list of strings.
2018 stb = value._render_traceback_()
2018 stb = value._render_traceback_()
2019 except Exception:
2019 except Exception:
2020 stb = self.InteractiveTB.structured_traceback(etype,
2020 stb = self.InteractiveTB.structured_traceback(etype,
2021 value, tb, tb_offset=tb_offset)
2021 value, tb, tb_offset=tb_offset)
2022
2022
2023 self._showtraceback(etype, value, stb)
2023 self._showtraceback(etype, value, stb)
2024 if self.call_pdb:
2024 if self.call_pdb:
2025 # drop into debugger
2025 # drop into debugger
2026 self.debugger(force=True)
2026 self.debugger(force=True)
2027 return
2027 return
2028
2028
2029 # Actually show the traceback
2029 # Actually show the traceback
2030 self._showtraceback(etype, value, stb)
2030 self._showtraceback(etype, value, stb)
2031
2031
2032 except KeyboardInterrupt:
2032 except KeyboardInterrupt:
2033 print('\n' + self.get_exception_only(), file=sys.stderr)
2033 print('\n' + self.get_exception_only(), file=sys.stderr)
2034
2034
2035 def _showtraceback(self, etype, evalue, stb):
2035 def _showtraceback(self, etype, evalue, stb):
2036 """Actually show a traceback.
2036 """Actually show a traceback.
2037
2037
2038 Subclasses may override this method to put the traceback on a different
2038 Subclasses may override this method to put the traceback on a different
2039 place, like a side channel.
2039 place, like a side channel.
2040 """
2040 """
2041 print(self.InteractiveTB.stb2text(stb))
2041 print(self.InteractiveTB.stb2text(stb))
2042
2042
2043 def showsyntaxerror(self, filename=None, running_compiled_code=False):
2043 def showsyntaxerror(self, filename=None, running_compiled_code=False):
2044 """Display the syntax error that just occurred.
2044 """Display the syntax error that just occurred.
2045
2045
2046 This doesn't display a stack trace because there isn't one.
2046 This doesn't display a stack trace because there isn't one.
2047
2047
2048 If a filename is given, it is stuffed in the exception instead
2048 If a filename is given, it is stuffed in the exception instead
2049 of what was there before (because Python's parser always uses
2049 of what was there before (because Python's parser always uses
2050 "<string>" when reading from a string).
2050 "<string>" when reading from a string).
2051
2051
2052 If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
2052 If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
2053 longer stack trace will be displayed.
2053 longer stack trace will be displayed.
2054 """
2054 """
2055 etype, value, last_traceback = self._get_exc_info()
2055 etype, value, last_traceback = self._get_exc_info()
2056
2056
2057 if filename and issubclass(etype, SyntaxError):
2057 if filename and issubclass(etype, SyntaxError):
2058 try:
2058 try:
2059 value.filename = filename
2059 value.filename = filename
2060 except:
2060 except:
2061 # Not the format we expect; leave it alone
2061 # Not the format we expect; leave it alone
2062 pass
2062 pass
2063
2063
2064 # If the error occurred when executing compiled code, we should provide full stacktrace.
2064 # If the error occurred when executing compiled code, we should provide full stacktrace.
2065 elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
2065 elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
2066 stb = self.SyntaxTB.structured_traceback(etype, value, elist)
2066 stb = self.SyntaxTB.structured_traceback(etype, value, elist)
2067 self._showtraceback(etype, value, stb)
2067 self._showtraceback(etype, value, stb)
2068
2068
2069 # This is overridden in TerminalInteractiveShell to show a message about
2069 # This is overridden in TerminalInteractiveShell to show a message about
2070 # the %paste magic.
2070 # the %paste magic.
2071 def showindentationerror(self):
2071 def showindentationerror(self):
2072 """Called by _run_cell when there's an IndentationError in code entered
2072 """Called by _run_cell when there's an IndentationError in code entered
2073 at the prompt.
2073 at the prompt.
2074
2074
2075 This is overridden in TerminalInteractiveShell to show a message about
2075 This is overridden in TerminalInteractiveShell to show a message about
2076 the %paste magic."""
2076 the %paste magic."""
2077 self.showsyntaxerror()
2077 self.showsyntaxerror()
2078
2078
2079 #-------------------------------------------------------------------------
2079 #-------------------------------------------------------------------------
2080 # Things related to readline
2080 # Things related to readline
2081 #-------------------------------------------------------------------------
2081 #-------------------------------------------------------------------------
2082
2082
2083 def init_readline(self):
2083 def init_readline(self):
2084 """DEPRECATED
2084 """DEPRECATED
2085
2085
2086 Moved to terminal subclass, here only to simplify the init logic."""
2086 Moved to terminal subclass, here only to simplify the init logic."""
2087 # Set a number of methods that depend on readline to be no-op
2087 # Set a number of methods that depend on readline to be no-op
2088 warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
2088 warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
2089 DeprecationWarning, stacklevel=2)
2089 DeprecationWarning, stacklevel=2)
2090 self.set_custom_completer = no_op
2090 self.set_custom_completer = no_op
2091
2091
2092 @skip_doctest
2092 @skip_doctest
2093 def set_next_input(self, s, replace=False):
2093 def set_next_input(self, s, replace=False):
2094 """ Sets the 'default' input string for the next command line.
2094 """ Sets the 'default' input string for the next command line.
2095
2095
2096 Example::
2096 Example::
2097
2097
2098 In [1]: _ip.set_next_input("Hello Word")
2098 In [1]: _ip.set_next_input("Hello Word")
2099 In [2]: Hello Word_ # cursor is here
2099 In [2]: Hello Word_ # cursor is here
2100 """
2100 """
2101 self.rl_next_input = s
2101 self.rl_next_input = s
2102
2102
2103 def _indent_current_str(self):
2103 def _indent_current_str(self):
2104 """return the current level of indentation as a string"""
2104 """return the current level of indentation as a string"""
2105 return self.input_splitter.get_indent_spaces() * ' '
2105 return self.input_splitter.get_indent_spaces() * ' '
2106
2106
2107 #-------------------------------------------------------------------------
2107 #-------------------------------------------------------------------------
2108 # Things related to text completion
2108 # Things related to text completion
2109 #-------------------------------------------------------------------------
2109 #-------------------------------------------------------------------------
2110
2110
2111 def init_completer(self):
2111 def init_completer(self):
2112 """Initialize the completion machinery.
2112 """Initialize the completion machinery.
2113
2113
2114 This creates completion machinery that can be used by client code,
2114 This creates completion machinery that can be used by client code,
2115 either interactively in-process (typically triggered by the readline
2115 either interactively in-process (typically triggered by the readline
2116 library), programmatically (such as in test suites) or out-of-process
2116 library), programmatically (such as in test suites) or out-of-process
2117 (typically over the network by remote frontends).
2117 (typically over the network by remote frontends).
2118 """
2118 """
2119 from IPython.core.completer import IPCompleter
2119 from IPython.core.completer import IPCompleter
2120 from IPython.core.completerlib import (module_completer,
2120 from IPython.core.completerlib import (module_completer,
2121 magic_run_completer, cd_completer, reset_completer)
2121 magic_run_completer, cd_completer, reset_completer)
2122
2122
2123 self.Completer = IPCompleter(shell=self,
2123 self.Completer = IPCompleter(shell=self,
2124 namespace=self.user_ns,
2124 namespace=self.user_ns,
2125 global_namespace=self.user_global_ns,
2125 global_namespace=self.user_global_ns,
2126 parent=self,
2126 parent=self,
2127 )
2127 )
2128 self.configurables.append(self.Completer)
2128 self.configurables.append(self.Completer)
2129
2129
2130 # Add custom completers to the basic ones built into IPCompleter
2130 # Add custom completers to the basic ones built into IPCompleter
2131 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
2131 sdisp = self.strdispatchers.get('complete_command', StrDispatch())
2132 self.strdispatchers['complete_command'] = sdisp
2132 self.strdispatchers['complete_command'] = sdisp
2133 self.Completer.custom_completers = sdisp
2133 self.Completer.custom_completers = sdisp
2134
2134
2135 self.set_hook('complete_command', module_completer, str_key = 'import')
2135 self.set_hook('complete_command', module_completer, str_key = 'import')
2136 self.set_hook('complete_command', module_completer, str_key = 'from')
2136 self.set_hook('complete_command', module_completer, str_key = 'from')
2137 self.set_hook('complete_command', module_completer, str_key = '%aimport')
2137 self.set_hook('complete_command', module_completer, str_key = '%aimport')
2138 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
2138 self.set_hook('complete_command', magic_run_completer, str_key = '%run')
2139 self.set_hook('complete_command', cd_completer, str_key = '%cd')
2139 self.set_hook('complete_command', cd_completer, str_key = '%cd')
2140 self.set_hook('complete_command', reset_completer, str_key = '%reset')
2140 self.set_hook('complete_command', reset_completer, str_key = '%reset')
2141
2141
2142 @skip_doctest
2142 @skip_doctest
2143 def complete(self, text, line=None, cursor_pos=None):
2143 def complete(self, text, line=None, cursor_pos=None):
2144 """Return the completed text and a list of completions.
2144 """Return the completed text and a list of completions.
2145
2145
2146 Parameters
2146 Parameters
2147 ----------
2147 ----------
2148
2148
2149 text : string
2149 text : string
2150 A string of text to be completed on. It can be given as empty and
2150 A string of text to be completed on. It can be given as empty and
2151 instead a line/position pair are given. In this case, the
2151 instead a line/position pair are given. In this case, the
2152 completer itself will split the line like readline does.
2152 completer itself will split the line like readline does.
2153
2153
2154 line : string, optional
2154 line : string, optional
2155 The complete line that text is part of.
2155 The complete line that text is part of.
2156
2156
2157 cursor_pos : int, optional
2157 cursor_pos : int, optional
2158 The position of the cursor on the input line.
2158 The position of the cursor on the input line.
2159
2159
2160 Returns
2160 Returns
2161 -------
2161 -------
2162 text : string
2162 text : string
2163 The actual text that was completed.
2163 The actual text that was completed.
2164
2164
2165 matches : list
2165 matches : list
2166 A sorted list with all possible completions.
2166 A sorted list with all possible completions.
2167
2167
2168 The optional arguments allow the completion to take more context into
2168 The optional arguments allow the completion to take more context into
2169 account, and are part of the low-level completion API.
2169 account, and are part of the low-level completion API.
2170
2170
2171 This is a wrapper around the completion mechanism, similar to what
2171 This is a wrapper around the completion mechanism, similar to what
2172 readline does at the command line when the TAB key is hit. By
2172 readline does at the command line when the TAB key is hit. By
2173 exposing it as a method, it can be used by other non-readline
2173 exposing it as a method, it can be used by other non-readline
2174 environments (such as GUIs) for text completion.
2174 environments (such as GUIs) for text completion.
2175
2175
2176 Simple usage example:
2176 Simple usage example:
2177
2177
2178 In [1]: x = 'hello'
2178 In [1]: x = 'hello'
2179
2179
2180 In [2]: _ip.complete('x.l')
2180 In [2]: _ip.complete('x.l')
2181 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
2181 Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
2182 """
2182 """
2183
2183
2184 # Inject names into __builtin__ so we can complete on the added names.
2184 # Inject names into __builtin__ so we can complete on the added names.
2185 with self.builtin_trap:
2185 with self.builtin_trap:
2186 return self.Completer.complete(text, line, cursor_pos)
2186 return self.Completer.complete(text, line, cursor_pos)
2187
2187
2188 def set_custom_completer(self, completer, pos=0):
2188 def set_custom_completer(self, completer, pos=0):
2189 """Adds a new custom completer function.
2189 """Adds a new custom completer function.
2190
2190
2191 The position argument (defaults to 0) is the index in the completers
2191 The position argument (defaults to 0) is the index in the completers
2192 list where you want the completer to be inserted."""
2192 list where you want the completer to be inserted."""
2193
2193
2194 newcomp = types.MethodType(completer,self.Completer)
2194 newcomp = types.MethodType(completer,self.Completer)
2195 self.Completer.matchers.insert(pos,newcomp)
2195 self.Completer.matchers.insert(pos,newcomp)
2196
2196
2197 def set_completer_frame(self, frame=None):
2197 def set_completer_frame(self, frame=None):
2198 """Set the frame of the completer."""
2198 """Set the frame of the completer."""
2199 if frame:
2199 if frame:
2200 self.Completer.namespace = frame.f_locals
2200 self.Completer.namespace = frame.f_locals
2201 self.Completer.global_namespace = frame.f_globals
2201 self.Completer.global_namespace = frame.f_globals
2202 else:
2202 else:
2203 self.Completer.namespace = self.user_ns
2203 self.Completer.namespace = self.user_ns
2204 self.Completer.global_namespace = self.user_global_ns
2204 self.Completer.global_namespace = self.user_global_ns
2205
2205
2206 #-------------------------------------------------------------------------
2206 #-------------------------------------------------------------------------
2207 # Things related to magics
2207 # Things related to magics
2208 #-------------------------------------------------------------------------
2208 #-------------------------------------------------------------------------
2209
2209
2210 def init_magics(self):
2210 def init_magics(self):
2211 from IPython.core import magics as m
2211 from IPython.core import magics as m
2212 self.magics_manager = magic.MagicsManager(shell=self,
2212 self.magics_manager = magic.MagicsManager(shell=self,
2213 parent=self,
2213 parent=self,
2214 user_magics=m.UserMagics(self))
2214 user_magics=m.UserMagics(self))
2215 self.configurables.append(self.magics_manager)
2215 self.configurables.append(self.magics_manager)
2216
2216
2217 # Expose as public API from the magics manager
2217 # Expose as public API from the magics manager
2218 self.register_magics = self.magics_manager.register
2218 self.register_magics = self.magics_manager.register
2219
2219
2220 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2220 self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
2221 m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
2221 m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
2222 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2222 m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
2223 m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
2223 m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
2224 m.PylabMagics, m.ScriptMagics,
2224 m.PylabMagics, m.ScriptMagics,
2225 )
2225 )
2226 if sys.version_info >(3,5):
2226 if sys.version_info >(3,5):
2227 self.register_magics(m.AsyncMagics)
2227 self.register_magics(m.AsyncMagics)
2228
2228
2229 # Register Magic Aliases
2229 # Register Magic Aliases
2230 mman = self.magics_manager
2230 mman = self.magics_manager
2231 # FIXME: magic aliases should be defined by the Magics classes
2231 # FIXME: magic aliases should be defined by the Magics classes
2232 # or in MagicsManager, not here
2232 # or in MagicsManager, not here
2233 mman.register_alias('ed', 'edit')
2233 mman.register_alias('ed', 'edit')
2234 mman.register_alias('hist', 'history')
2234 mman.register_alias('hist', 'history')
2235 mman.register_alias('rep', 'recall')
2235 mman.register_alias('rep', 'recall')
2236 mman.register_alias('SVG', 'svg', 'cell')
2236 mman.register_alias('SVG', 'svg', 'cell')
2237 mman.register_alias('HTML', 'html', 'cell')
2237 mman.register_alias('HTML', 'html', 'cell')
2238 mman.register_alias('file', 'writefile', 'cell')
2238 mman.register_alias('file', 'writefile', 'cell')
2239
2239
2240 # FIXME: Move the color initialization to the DisplayHook, which
2240 # FIXME: Move the color initialization to the DisplayHook, which
2241 # should be split into a prompt manager and displayhook. We probably
2241 # should be split into a prompt manager and displayhook. We probably
2242 # even need a centralize colors management object.
2242 # even need a centralize colors management object.
2243 self.run_line_magic('colors', self.colors)
2243 self.run_line_magic('colors', self.colors)
2244
2244
2245 # Defined here so that it's included in the documentation
2245 # Defined here so that it's included in the documentation
2246 @functools.wraps(magic.MagicsManager.register_function)
2246 @functools.wraps(magic.MagicsManager.register_function)
2247 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2247 def register_magic_function(self, func, magic_kind='line', magic_name=None):
2248 self.magics_manager.register_function(func,
2248 self.magics_manager.register_function(func,
2249 magic_kind=magic_kind, magic_name=magic_name)
2249 magic_kind=magic_kind, magic_name=magic_name)
2250
2250
2251 def run_line_magic(self, magic_name, line, _stack_depth=1):
2251 def run_line_magic(self, magic_name, line, _stack_depth=1):
2252 """Execute the given line magic.
2252 """Execute the given line magic.
2253
2253
2254 Parameters
2254 Parameters
2255 ----------
2255 ----------
2256 magic_name : str
2256 magic_name : str
2257 Name of the desired magic function, without '%' prefix.
2257 Name of the desired magic function, without '%' prefix.
2258
2258
2259 line : str
2259 line : str
2260 The rest of the input line as a single string.
2260 The rest of the input line as a single string.
2261
2261
2262 _stack_depth : int
2262 _stack_depth : int
2263 If run_line_magic() is called from magic() then _stack_depth=2.
2263 If run_line_magic() is called from magic() then _stack_depth=2.
2264 This is added to ensure backward compatibility for use of 'get_ipython().magic()'
2264 This is added to ensure backward compatibility for use of 'get_ipython().magic()'
2265 """
2265 """
2266 fn = self.find_line_magic(magic_name)
2266 fn = self.find_line_magic(magic_name)
2267 if fn is None:
2267 if fn is None:
2268 cm = self.find_cell_magic(magic_name)
2268 cm = self.find_cell_magic(magic_name)
2269 etpl = "Line magic function `%%%s` not found%s."
2269 etpl = "Line magic function `%%%s` not found%s."
2270 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2270 extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
2271 'did you mean that instead?)' % magic_name )
2271 'did you mean that instead?)' % magic_name )
2272 raise UsageError(etpl % (magic_name, extra))
2272 raise UsageError(etpl % (magic_name, extra))
2273 else:
2273 else:
2274 # Note: this is the distance in the stack to the user's frame.
2274 # Note: this is the distance in the stack to the user's frame.
2275 # This will need to be updated if the internal calling logic gets
2275 # This will need to be updated if the internal calling logic gets
2276 # refactored, or else we'll be expanding the wrong variables.
2276 # refactored, or else we'll be expanding the wrong variables.
2277
2277
2278 # Determine stack_depth depending on where run_line_magic() has been called
2278 # Determine stack_depth depending on where run_line_magic() has been called
2279 stack_depth = _stack_depth
2279 stack_depth = _stack_depth
2280 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2280 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2281 # magic has opted out of var_expand
2281 # magic has opted out of var_expand
2282 magic_arg_s = line
2282 magic_arg_s = line
2283 else:
2283 else:
2284 magic_arg_s = self.var_expand(line, stack_depth)
2284 magic_arg_s = self.var_expand(line, stack_depth)
2285 # Put magic args in a list so we can call with f(*a) syntax
2285 # Put magic args in a list so we can call with f(*a) syntax
2286 args = [magic_arg_s]
2286 args = [magic_arg_s]
2287 kwargs = {}
2287 kwargs = {}
2288 # Grab local namespace if we need it:
2288 # Grab local namespace if we need it:
2289 if getattr(fn, "needs_local_scope", False):
2289 if getattr(fn, "needs_local_scope", False):
2290 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2290 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
2291 with self.builtin_trap:
2291 with self.builtin_trap:
2292 result = fn(*args, **kwargs)
2292 result = fn(*args, **kwargs)
2293 return result
2293 return result
2294
2294
2295 def run_cell_magic(self, magic_name, line, cell):
2295 def run_cell_magic(self, magic_name, line, cell):
2296 """Execute the given cell magic.
2296 """Execute the given cell magic.
2297
2297
2298 Parameters
2298 Parameters
2299 ----------
2299 ----------
2300 magic_name : str
2300 magic_name : str
2301 Name of the desired magic function, without '%' prefix.
2301 Name of the desired magic function, without '%' prefix.
2302
2302
2303 line : str
2303 line : str
2304 The rest of the first input line as a single string.
2304 The rest of the first input line as a single string.
2305
2305
2306 cell : str
2306 cell : str
2307 The body of the cell as a (possibly multiline) string.
2307 The body of the cell as a (possibly multiline) string.
2308 """
2308 """
2309 fn = self.find_cell_magic(magic_name)
2309 fn = self.find_cell_magic(magic_name)
2310 if fn is None:
2310 if fn is None:
2311 lm = self.find_line_magic(magic_name)
2311 lm = self.find_line_magic(magic_name)
2312 etpl = "Cell magic `%%{0}` not found{1}."
2312 etpl = "Cell magic `%%{0}` not found{1}."
2313 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2313 extra = '' if lm is None else (' (But line magic `%{0}` exists, '
2314 'did you mean that instead?)'.format(magic_name))
2314 'did you mean that instead?)'.format(magic_name))
2315 raise UsageError(etpl.format(magic_name, extra))
2315 raise UsageError(etpl.format(magic_name, extra))
2316 elif cell == '':
2316 elif cell == '':
2317 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2317 message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
2318 if self.find_line_magic(magic_name) is not None:
2318 if self.find_line_magic(magic_name) is not None:
2319 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2319 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
2320 raise UsageError(message)
2320 raise UsageError(message)
2321 else:
2321 else:
2322 # Note: this is the distance in the stack to the user's frame.
2322 # Note: this is the distance in the stack to the user's frame.
2323 # This will need to be updated if the internal calling logic gets
2323 # This will need to be updated if the internal calling logic gets
2324 # refactored, or else we'll be expanding the wrong variables.
2324 # refactored, or else we'll be expanding the wrong variables.
2325 stack_depth = 2
2325 stack_depth = 2
2326 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2326 if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
2327 # magic has opted out of var_expand
2327 # magic has opted out of var_expand
2328 magic_arg_s = line
2328 magic_arg_s = line
2329 else:
2329 else:
2330 magic_arg_s = self.var_expand(line, stack_depth)
2330 magic_arg_s = self.var_expand(line, stack_depth)
2331 with self.builtin_trap:
2331 with self.builtin_trap:
2332 result = fn(magic_arg_s, cell)
2332 result = fn(magic_arg_s, cell)
2333 return result
2333 return result
2334
2334
2335 def find_line_magic(self, magic_name):
2335 def find_line_magic(self, magic_name):
2336 """Find and return a line magic by name.
2336 """Find and return a line magic by name.
2337
2337
2338 Returns None if the magic isn't found."""
2338 Returns None if the magic isn't found."""
2339 return self.magics_manager.magics['line'].get(magic_name)
2339 return self.magics_manager.magics['line'].get(magic_name)
2340
2340
2341 def find_cell_magic(self, magic_name):
2341 def find_cell_magic(self, magic_name):
2342 """Find and return a cell magic by name.
2342 """Find and return a cell magic by name.
2343
2343
2344 Returns None if the magic isn't found."""
2344 Returns None if the magic isn't found."""
2345 return self.magics_manager.magics['cell'].get(magic_name)
2345 return self.magics_manager.magics['cell'].get(magic_name)
2346
2346
2347 def find_magic(self, magic_name, magic_kind='line'):
2347 def find_magic(self, magic_name, magic_kind='line'):
2348 """Find and return a magic of the given type by name.
2348 """Find and return a magic of the given type by name.
2349
2349
2350 Returns None if the magic isn't found."""
2350 Returns None if the magic isn't found."""
2351 return self.magics_manager.magics[magic_kind].get(magic_name)
2351 return self.magics_manager.magics[magic_kind].get(magic_name)
2352
2352
2353 def magic(self, arg_s):
2353 def magic(self, arg_s):
2354 """DEPRECATED. Use run_line_magic() instead.
2354 """DEPRECATED. Use run_line_magic() instead.
2355
2355
2356 Call a magic function by name.
2356 Call a magic function by name.
2357
2357
2358 Input: a string containing the name of the magic function to call and
2358 Input: a string containing the name of the magic function to call and
2359 any additional arguments to be passed to the magic.
2359 any additional arguments to be passed to the magic.
2360
2360
2361 magic('name -opt foo bar') is equivalent to typing at the ipython
2361 magic('name -opt foo bar') is equivalent to typing at the ipython
2362 prompt:
2362 prompt:
2363
2363
2364 In[1]: %name -opt foo bar
2364 In[1]: %name -opt foo bar
2365
2365
2366 To call a magic without arguments, simply use magic('name').
2366 To call a magic without arguments, simply use magic('name').
2367
2367
2368 This provides a proper Python function to call IPython's magics in any
2368 This provides a proper Python function to call IPython's magics in any
2369 valid Python code you can type at the interpreter, including loops and
2369 valid Python code you can type at the interpreter, including loops and
2370 compound statements.
2370 compound statements.
2371 """
2371 """
2372 # TODO: should we issue a loud deprecation warning here?
2372 # TODO: should we issue a loud deprecation warning here?
2373 magic_name, _, magic_arg_s = arg_s.partition(' ')
2373 magic_name, _, magic_arg_s = arg_s.partition(' ')
2374 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2374 magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
2375 return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
2375 return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
2376
2376
2377 #-------------------------------------------------------------------------
2377 #-------------------------------------------------------------------------
2378 # Things related to macros
2378 # Things related to macros
2379 #-------------------------------------------------------------------------
2379 #-------------------------------------------------------------------------
2380
2380
2381 def define_macro(self, name, themacro):
2381 def define_macro(self, name, themacro):
2382 """Define a new macro
2382 """Define a new macro
2383
2383
2384 Parameters
2384 Parameters
2385 ----------
2385 ----------
2386 name : str
2386 name : str
2387 The name of the macro.
2387 The name of the macro.
2388 themacro : str or Macro
2388 themacro : str or Macro
2389 The action to do upon invoking the macro. If a string, a new
2389 The action to do upon invoking the macro. If a string, a new
2390 Macro object is created by passing the string to it.
2390 Macro object is created by passing the string to it.
2391 """
2391 """
2392
2392
2393 from IPython.core import macro
2393 from IPython.core import macro
2394
2394
2395 if isinstance(themacro, str):
2395 if isinstance(themacro, str):
2396 themacro = macro.Macro(themacro)
2396 themacro = macro.Macro(themacro)
2397 if not isinstance(themacro, macro.Macro):
2397 if not isinstance(themacro, macro.Macro):
2398 raise ValueError('A macro must be a string or a Macro instance.')
2398 raise ValueError('A macro must be a string or a Macro instance.')
2399 self.user_ns[name] = themacro
2399 self.user_ns[name] = themacro
2400
2400
2401 #-------------------------------------------------------------------------
2401 #-------------------------------------------------------------------------
2402 # Things related to the running of system commands
2402 # Things related to the running of system commands
2403 #-------------------------------------------------------------------------
2403 #-------------------------------------------------------------------------
2404
2404
2405 def system_piped(self, cmd):
2405 def system_piped(self, cmd):
2406 """Call the given cmd in a subprocess, piping stdout/err
2406 """Call the given cmd in a subprocess, piping stdout/err
2407
2407
2408 Parameters
2408 Parameters
2409 ----------
2409 ----------
2410 cmd : str
2410 cmd : str
2411 Command to execute (can not end in '&', as background processes are
2411 Command to execute (can not end in '&', as background processes are
2412 not supported. Should not be a command that expects input
2412 not supported. Should not be a command that expects input
2413 other than simple text.
2413 other than simple text.
2414 """
2414 """
2415 if cmd.rstrip().endswith('&'):
2415 if cmd.rstrip().endswith('&'):
2416 # this is *far* from a rigorous test
2416 # this is *far* from a rigorous test
2417 # We do not support backgrounding processes because we either use
2417 # We do not support backgrounding processes because we either use
2418 # pexpect or pipes to read from. Users can always just call
2418 # pexpect or pipes to read from. Users can always just call
2419 # os.system() or use ip.system=ip.system_raw
2419 # os.system() or use ip.system=ip.system_raw
2420 # if they really want a background process.
2420 # if they really want a background process.
2421 raise OSError("Background processes not supported.")
2421 raise OSError("Background processes not supported.")
2422
2422
2423 # we explicitly do NOT return the subprocess status code, because
2423 # we explicitly do NOT return the subprocess status code, because
2424 # a non-None value would trigger :func:`sys.displayhook` calls.
2424 # a non-None value would trigger :func:`sys.displayhook` calls.
2425 # Instead, we store the exit_code in user_ns.
2425 # Instead, we store the exit_code in user_ns.
2426 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2426 self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
2427
2427
2428 def system_raw(self, cmd):
2428 def system_raw(self, cmd):
2429 """Call the given cmd in a subprocess using os.system on Windows or
2429 """Call the given cmd in a subprocess using os.system on Windows or
2430 subprocess.call using the system shell on other platforms.
2430 subprocess.call using the system shell on other platforms.
2431
2431
2432 Parameters
2432 Parameters
2433 ----------
2433 ----------
2434 cmd : str
2434 cmd : str
2435 Command to execute.
2435 Command to execute.
2436 """
2436 """
2437 cmd = self.var_expand(cmd, depth=1)
2437 cmd = self.var_expand(cmd, depth=1)
2438 # protect os.system from UNC paths on Windows, which it can't handle:
2438 # protect os.system from UNC paths on Windows, which it can't handle:
2439 if sys.platform == 'win32':
2439 if sys.platform == 'win32':
2440 from IPython.utils._process_win32 import AvoidUNCPath
2440 from IPython.utils._process_win32 import AvoidUNCPath
2441 with AvoidUNCPath() as path:
2441 with AvoidUNCPath() as path:
2442 if path is not None:
2442 if path is not None:
2443 cmd = '"pushd %s &&"%s' % (path, cmd)
2443 cmd = '"pushd %s &&"%s' % (path, cmd)
2444 try:
2444 try:
2445 ec = os.system(cmd)
2445 ec = os.system(cmd)
2446 except KeyboardInterrupt:
2446 except KeyboardInterrupt:
2447 print('\n' + self.get_exception_only(), file=sys.stderr)
2447 print('\n' + self.get_exception_only(), file=sys.stderr)
2448 ec = -2
2448 ec = -2
2449 else:
2449 else:
2450 # For posix the result of the subprocess.call() below is an exit
2450 # For posix the result of the subprocess.call() below is an exit
2451 # code, which by convention is zero for success, positive for
2451 # code, which by convention is zero for success, positive for
2452 # program failure. Exit codes above 128 are reserved for signals,
2452 # program failure. Exit codes above 128 are reserved for signals,
2453 # and the formula for converting a signal to an exit code is usually
2453 # and the formula for converting a signal to an exit code is usually
2454 # signal_number+128. To more easily differentiate between exit
2454 # signal_number+128. To more easily differentiate between exit
2455 # codes and signals, ipython uses negative numbers. For instance
2455 # codes and signals, ipython uses negative numbers. For instance
2456 # since control-c is signal 2 but exit code 130, ipython's
2456 # since control-c is signal 2 but exit code 130, ipython's
2457 # _exit_code variable will read -2. Note that some shells like
2457 # _exit_code variable will read -2. Note that some shells like
2458 # csh and fish don't follow sh/bash conventions for exit codes.
2458 # csh and fish don't follow sh/bash conventions for exit codes.
2459 executable = os.environ.get('SHELL', None)
2459 executable = os.environ.get('SHELL', None)
2460 try:
2460 try:
2461 # Use env shell instead of default /bin/sh
2461 # Use env shell instead of default /bin/sh
2462 ec = subprocess.call(cmd, shell=True, executable=executable)
2462 ec = subprocess.call(cmd, shell=True, executable=executable)
2463 except KeyboardInterrupt:
2463 except KeyboardInterrupt:
2464 # intercept control-C; a long traceback is not useful here
2464 # intercept control-C; a long traceback is not useful here
2465 print('\n' + self.get_exception_only(), file=sys.stderr)
2465 print('\n' + self.get_exception_only(), file=sys.stderr)
2466 ec = 130
2466 ec = 130
2467 if ec > 128:
2467 if ec > 128:
2468 ec = -(ec - 128)
2468 ec = -(ec - 128)
2469
2469
2470 # We explicitly do NOT return the subprocess status code, because
2470 # We explicitly do NOT return the subprocess status code, because
2471 # a non-None value would trigger :func:`sys.displayhook` calls.
2471 # a non-None value would trigger :func:`sys.displayhook` calls.
2472 # Instead, we store the exit_code in user_ns. Note the semantics
2472 # Instead, we store the exit_code in user_ns. Note the semantics
2473 # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
2473 # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
2474 # but raising SystemExit(_exit_code) will give status 254!
2474 # but raising SystemExit(_exit_code) will give status 254!
2475 self.user_ns['_exit_code'] = ec
2475 self.user_ns['_exit_code'] = ec
2476
2476
2477 # use piped system by default, because it is better behaved
2477 # use piped system by default, because it is better behaved
2478 system = system_piped
2478 system = system_piped
2479
2479
2480 def getoutput(self, cmd, split=True, depth=0):
2480 def getoutput(self, cmd, split=True, depth=0):
2481 """Get output (possibly including stderr) from a subprocess.
2481 """Get output (possibly including stderr) from a subprocess.
2482
2482
2483 Parameters
2483 Parameters
2484 ----------
2484 ----------
2485 cmd : str
2485 cmd : str
2486 Command to execute (can not end in '&', as background processes are
2486 Command to execute (can not end in '&', as background processes are
2487 not supported.
2487 not supported.
2488 split : bool, optional
2488 split : bool, optional
2489 If True, split the output into an IPython SList. Otherwise, an
2489 If True, split the output into an IPython SList. Otherwise, an
2490 IPython LSString is returned. These are objects similar to normal
2490 IPython LSString is returned. These are objects similar to normal
2491 lists and strings, with a few convenience attributes for easier
2491 lists and strings, with a few convenience attributes for easier
2492 manipulation of line-based output. You can use '?' on them for
2492 manipulation of line-based output. You can use '?' on them for
2493 details.
2493 details.
2494 depth : int, optional
2494 depth : int, optional
2495 How many frames above the caller are the local variables which should
2495 How many frames above the caller are the local variables which should
2496 be expanded in the command string? The default (0) assumes that the
2496 be expanded in the command string? The default (0) assumes that the
2497 expansion variables are in the stack frame calling this function.
2497 expansion variables are in the stack frame calling this function.
2498 """
2498 """
2499 if cmd.rstrip().endswith('&'):
2499 if cmd.rstrip().endswith('&'):
2500 # this is *far* from a rigorous test
2500 # this is *far* from a rigorous test
2501 raise OSError("Background processes not supported.")
2501 raise OSError("Background processes not supported.")
2502 out = getoutput(self.var_expand(cmd, depth=depth+1))
2502 out = getoutput(self.var_expand(cmd, depth=depth+1))
2503 if split:
2503 if split:
2504 out = SList(out.splitlines())
2504 out = SList(out.splitlines())
2505 else:
2505 else:
2506 out = LSString(out)
2506 out = LSString(out)
2507 return out
2507 return out
2508
2508
2509 #-------------------------------------------------------------------------
2509 #-------------------------------------------------------------------------
2510 # Things related to aliases
2510 # Things related to aliases
2511 #-------------------------------------------------------------------------
2511 #-------------------------------------------------------------------------
2512
2512
2513 def init_alias(self):
2513 def init_alias(self):
2514 self.alias_manager = AliasManager(shell=self, parent=self)
2514 self.alias_manager = AliasManager(shell=self, parent=self)
2515 self.configurables.append(self.alias_manager)
2515 self.configurables.append(self.alias_manager)
2516
2516
2517 #-------------------------------------------------------------------------
2517 #-------------------------------------------------------------------------
2518 # Things related to extensions
2518 # Things related to extensions
2519 #-------------------------------------------------------------------------
2519 #-------------------------------------------------------------------------
2520
2520
2521 def init_extension_manager(self):
2521 def init_extension_manager(self):
2522 self.extension_manager = ExtensionManager(shell=self, parent=self)
2522 self.extension_manager = ExtensionManager(shell=self, parent=self)
2523 self.configurables.append(self.extension_manager)
2523 self.configurables.append(self.extension_manager)
2524
2524
2525 #-------------------------------------------------------------------------
2525 #-------------------------------------------------------------------------
2526 # Things related to payloads
2526 # Things related to payloads
2527 #-------------------------------------------------------------------------
2527 #-------------------------------------------------------------------------
2528
2528
2529 def init_payload(self):
2529 def init_payload(self):
2530 self.payload_manager = PayloadManager(parent=self)
2530 self.payload_manager = PayloadManager(parent=self)
2531 self.configurables.append(self.payload_manager)
2531 self.configurables.append(self.payload_manager)
2532
2532
2533 #-------------------------------------------------------------------------
2533 #-------------------------------------------------------------------------
2534 # Things related to the prefilter
2534 # Things related to the prefilter
2535 #-------------------------------------------------------------------------
2535 #-------------------------------------------------------------------------
2536
2536
2537 def init_prefilter(self):
2537 def init_prefilter(self):
2538 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2538 self.prefilter_manager = PrefilterManager(shell=self, parent=self)
2539 self.configurables.append(self.prefilter_manager)
2539 self.configurables.append(self.prefilter_manager)
2540 # Ultimately this will be refactored in the new interpreter code, but
2540 # Ultimately this will be refactored in the new interpreter code, but
2541 # for now, we should expose the main prefilter method (there's legacy
2541 # for now, we should expose the main prefilter method (there's legacy
2542 # code out there that may rely on this).
2542 # code out there that may rely on this).
2543 self.prefilter = self.prefilter_manager.prefilter_lines
2543 self.prefilter = self.prefilter_manager.prefilter_lines
2544
2544
2545 def auto_rewrite_input(self, cmd):
2545 def auto_rewrite_input(self, cmd):
2546 """Print to the screen the rewritten form of the user's command.
2546 """Print to the screen the rewritten form of the user's command.
2547
2547
2548 This shows visual feedback by rewriting input lines that cause
2548 This shows visual feedback by rewriting input lines that cause
2549 automatic calling to kick in, like::
2549 automatic calling to kick in, like::
2550
2550
2551 /f x
2551 /f x
2552
2552
2553 into::
2553 into::
2554
2554
2555 ------> f(x)
2555 ------> f(x)
2556
2556
2557 after the user's input prompt. This helps the user understand that the
2557 after the user's input prompt. This helps the user understand that the
2558 input line was transformed automatically by IPython.
2558 input line was transformed automatically by IPython.
2559 """
2559 """
2560 if not self.show_rewritten_input:
2560 if not self.show_rewritten_input:
2561 return
2561 return
2562
2562
2563 # This is overridden in TerminalInteractiveShell to use fancy prompts
2563 # This is overridden in TerminalInteractiveShell to use fancy prompts
2564 print("------> " + cmd)
2564 print("------> " + cmd)
2565
2565
2566 #-------------------------------------------------------------------------
2566 #-------------------------------------------------------------------------
2567 # Things related to extracting values/expressions from kernel and user_ns
2567 # Things related to extracting values/expressions from kernel and user_ns
2568 #-------------------------------------------------------------------------
2568 #-------------------------------------------------------------------------
2569
2569
2570 def _user_obj_error(self):
2570 def _user_obj_error(self):
2571 """return simple exception dict
2571 """return simple exception dict
2572
2572
2573 for use in user_expressions
2573 for use in user_expressions
2574 """
2574 """
2575
2575
2576 etype, evalue, tb = self._get_exc_info()
2576 etype, evalue, tb = self._get_exc_info()
2577 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2577 stb = self.InteractiveTB.get_exception_only(etype, evalue)
2578
2578
2579 exc_info = {
2579 exc_info = {
2580 u'status' : 'error',
2580 u'status' : 'error',
2581 u'traceback' : stb,
2581 u'traceback' : stb,
2582 u'ename' : etype.__name__,
2582 u'ename' : etype.__name__,
2583 u'evalue' : py3compat.safe_unicode(evalue),
2583 u'evalue' : py3compat.safe_unicode(evalue),
2584 }
2584 }
2585
2585
2586 return exc_info
2586 return exc_info
2587
2587
2588 def _format_user_obj(self, obj):
2588 def _format_user_obj(self, obj):
2589 """format a user object to display dict
2589 """format a user object to display dict
2590
2590
2591 for use in user_expressions
2591 for use in user_expressions
2592 """
2592 """
2593
2593
2594 data, md = self.display_formatter.format(obj)
2594 data, md = self.display_formatter.format(obj)
2595 value = {
2595 value = {
2596 'status' : 'ok',
2596 'status' : 'ok',
2597 'data' : data,
2597 'data' : data,
2598 'metadata' : md,
2598 'metadata' : md,
2599 }
2599 }
2600 return value
2600 return value
2601
2601
2602 def user_expressions(self, expressions):
2602 def user_expressions(self, expressions):
2603 """Evaluate a dict of expressions in the user's namespace.
2603 """Evaluate a dict of expressions in the user's namespace.
2604
2604
2605 Parameters
2605 Parameters
2606 ----------
2606 ----------
2607 expressions : dict
2607 expressions : dict
2608 A dict with string keys and string values. The expression values
2608 A dict with string keys and string values. The expression values
2609 should be valid Python expressions, each of which will be evaluated
2609 should be valid Python expressions, each of which will be evaluated
2610 in the user namespace.
2610 in the user namespace.
2611
2611
2612 Returns
2612 Returns
2613 -------
2613 -------
2614 A dict, keyed like the input expressions dict, with the rich mime-typed
2614 A dict, keyed like the input expressions dict, with the rich mime-typed
2615 display_data of each value.
2615 display_data of each value.
2616 """
2616 """
2617 out = {}
2617 out = {}
2618 user_ns = self.user_ns
2618 user_ns = self.user_ns
2619 global_ns = self.user_global_ns
2619 global_ns = self.user_global_ns
2620
2620
2621 for key, expr in expressions.items():
2621 for key, expr in expressions.items():
2622 try:
2622 try:
2623 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2623 value = self._format_user_obj(eval(expr, global_ns, user_ns))
2624 except:
2624 except:
2625 value = self._user_obj_error()
2625 value = self._user_obj_error()
2626 out[key] = value
2626 out[key] = value
2627 return out
2627 return out
2628
2628
2629 #-------------------------------------------------------------------------
2629 #-------------------------------------------------------------------------
2630 # Things related to the running of code
2630 # Things related to the running of code
2631 #-------------------------------------------------------------------------
2631 #-------------------------------------------------------------------------
2632
2632
2633 def ex(self, cmd):
2633 def ex(self, cmd):
2634 """Execute a normal python statement in user namespace."""
2634 """Execute a normal python statement in user namespace."""
2635 with self.builtin_trap:
2635 with self.builtin_trap:
2636 exec(cmd, self.user_global_ns, self.user_ns)
2636 exec(cmd, self.user_global_ns, self.user_ns)
2637
2637
2638 def ev(self, expr):
2638 def ev(self, expr):
2639 """Evaluate python expression expr in user namespace.
2639 """Evaluate python expression expr in user namespace.
2640
2640
2641 Returns the result of evaluation
2641 Returns the result of evaluation
2642 """
2642 """
2643 with self.builtin_trap:
2643 with self.builtin_trap:
2644 return eval(expr, self.user_global_ns, self.user_ns)
2644 return eval(expr, self.user_global_ns, self.user_ns)
2645
2645
2646 def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
2646 def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
2647 """A safe version of the builtin execfile().
2647 """A safe version of the builtin execfile().
2648
2648
2649 This version will never throw an exception, but instead print
2649 This version will never throw an exception, but instead print
2650 helpful error messages to the screen. This only works on pure
2650 helpful error messages to the screen. This only works on pure
2651 Python files with the .py extension.
2651 Python files with the .py extension.
2652
2652
2653 Parameters
2653 Parameters
2654 ----------
2654 ----------
2655 fname : string
2655 fname : string
2656 The name of the file to be executed.
2656 The name of the file to be executed.
2657 where : tuple
2657 where : tuple
2658 One or two namespaces, passed to execfile() as (globals,locals).
2658 One or two namespaces, passed to execfile() as (globals,locals).
2659 If only one is given, it is passed as both.
2659 If only one is given, it is passed as both.
2660 exit_ignore : bool (False)
2660 exit_ignore : bool (False)
2661 If True, then silence SystemExit for non-zero status (it is always
2661 If True, then silence SystemExit for non-zero status (it is always
2662 silenced for zero status, as it is so common).
2662 silenced for zero status, as it is so common).
2663 raise_exceptions : bool (False)
2663 raise_exceptions : bool (False)
2664 If True raise exceptions everywhere. Meant for testing.
2664 If True raise exceptions everywhere. Meant for testing.
2665 shell_futures : bool (False)
2665 shell_futures : bool (False)
2666 If True, the code will share future statements with the interactive
2666 If True, the code will share future statements with the interactive
2667 shell. It will both be affected by previous __future__ imports, and
2667 shell. It will both be affected by previous __future__ imports, and
2668 any __future__ imports in the code will affect the shell. If False,
2668 any __future__ imports in the code will affect the shell. If False,
2669 __future__ imports are not shared in either direction.
2669 __future__ imports are not shared in either direction.
2670
2670
2671 """
2671 """
2672 fname = os.path.abspath(os.path.expanduser(fname))
2672 fname = os.path.abspath(os.path.expanduser(fname))
2673
2673
2674 # Make sure we can open the file
2674 # Make sure we can open the file
2675 try:
2675 try:
2676 with open(fname):
2676 with open(fname):
2677 pass
2677 pass
2678 except:
2678 except:
2679 warn('Could not open file <%s> for safe execution.' % fname)
2679 warn('Could not open file <%s> for safe execution.' % fname)
2680 return
2680 return
2681
2681
2682 # Find things also in current directory. This is needed to mimic the
2682 # Find things also in current directory. This is needed to mimic the
2683 # behavior of running a script from the system command line, where
2683 # behavior of running a script from the system command line, where
2684 # Python inserts the script's directory into sys.path
2684 # Python inserts the script's directory into sys.path
2685 dname = os.path.dirname(fname)
2685 dname = os.path.dirname(fname)
2686
2686
2687 with prepended_to_syspath(dname), self.builtin_trap:
2687 with prepended_to_syspath(dname), self.builtin_trap:
2688 try:
2688 try:
2689 glob, loc = (where + (None, ))[:2]
2689 glob, loc = (where + (None, ))[:2]
2690 py3compat.execfile(
2690 py3compat.execfile(
2691 fname, glob, loc,
2691 fname, glob, loc,
2692 self.compile if shell_futures else None)
2692 self.compile if shell_futures else None)
2693 except SystemExit as status:
2693 except SystemExit as status:
2694 # If the call was made with 0 or None exit status (sys.exit(0)
2694 # If the call was made with 0 or None exit status (sys.exit(0)
2695 # or sys.exit() ), don't bother showing a traceback, as both of
2695 # or sys.exit() ), don't bother showing a traceback, as both of
2696 # these are considered normal by the OS:
2696 # these are considered normal by the OS:
2697 # > python -c'import sys;sys.exit(0)'; echo $?
2697 # > python -c'import sys;sys.exit(0)'; echo $?
2698 # 0
2698 # 0
2699 # > python -c'import sys;sys.exit()'; echo $?
2699 # > python -c'import sys;sys.exit()'; echo $?
2700 # 0
2700 # 0
2701 # For other exit status, we show the exception unless
2701 # For other exit status, we show the exception unless
2702 # explicitly silenced, but only in short form.
2702 # explicitly silenced, but only in short form.
2703 if status.code:
2703 if status.code:
2704 if raise_exceptions:
2704 if raise_exceptions:
2705 raise
2705 raise
2706 if not exit_ignore:
2706 if not exit_ignore:
2707 self.showtraceback(exception_only=True)
2707 self.showtraceback(exception_only=True)
2708 except:
2708 except:
2709 if raise_exceptions:
2709 if raise_exceptions:
2710 raise
2710 raise
2711 # tb offset is 2 because we wrap execfile
2711 # tb offset is 2 because we wrap execfile
2712 self.showtraceback(tb_offset=2)
2712 self.showtraceback(tb_offset=2)
2713
2713
2714 def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
2714 def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
2715 """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
2715 """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
2716
2716
2717 Parameters
2717 Parameters
2718 ----------
2718 ----------
2719 fname : str
2719 fname : str
2720 The name of the file to execute. The filename must have a
2720 The name of the file to execute. The filename must have a
2721 .ipy or .ipynb extension.
2721 .ipy or .ipynb extension.
2722 shell_futures : bool (False)
2722 shell_futures : bool (False)
2723 If True, the code will share future statements with the interactive
2723 If True, the code will share future statements with the interactive
2724 shell. It will both be affected by previous __future__ imports, and
2724 shell. It will both be affected by previous __future__ imports, and
2725 any __future__ imports in the code will affect the shell. If False,
2725 any __future__ imports in the code will affect the shell. If False,
2726 __future__ imports are not shared in either direction.
2726 __future__ imports are not shared in either direction.
2727 raise_exceptions : bool (False)
2727 raise_exceptions : bool (False)
2728 If True raise exceptions everywhere. Meant for testing.
2728 If True raise exceptions everywhere. Meant for testing.
2729 """
2729 """
2730 fname = os.path.abspath(os.path.expanduser(fname))
2730 fname = os.path.abspath(os.path.expanduser(fname))
2731
2731
2732 # Make sure we can open the file
2732 # Make sure we can open the file
2733 try:
2733 try:
2734 with open(fname):
2734 with open(fname):
2735 pass
2735 pass
2736 except:
2736 except:
2737 warn('Could not open file <%s> for safe execution.' % fname)
2737 warn('Could not open file <%s> for safe execution.' % fname)
2738 return
2738 return
2739
2739
2740 # Find things also in current directory. This is needed to mimic the
2740 # Find things also in current directory. This is needed to mimic the
2741 # behavior of running a script from the system command line, where
2741 # behavior of running a script from the system command line, where
2742 # Python inserts the script's directory into sys.path
2742 # Python inserts the script's directory into sys.path
2743 dname = os.path.dirname(fname)
2743 dname = os.path.dirname(fname)
2744
2744
2745 def get_cells():
2745 def get_cells():
2746 """generator for sequence of code blocks to run"""
2746 """generator for sequence of code blocks to run"""
2747 if fname.endswith('.ipynb'):
2747 if fname.endswith('.ipynb'):
2748 from nbformat import read
2748 from nbformat import read
2749 nb = read(fname, as_version=4)
2749 nb = read(fname, as_version=4)
2750 if not nb.cells:
2750 if not nb.cells:
2751 return
2751 return
2752 for cell in nb.cells:
2752 for cell in nb.cells:
2753 if cell.cell_type == 'code':
2753 if cell.cell_type == 'code':
2754 yield cell.source
2754 yield cell.source
2755 else:
2755 else:
2756 with open(fname) as f:
2756 with open(fname) as f:
2757 yield f.read()
2757 yield f.read()
2758
2758
2759 with prepended_to_syspath(dname):
2759 with prepended_to_syspath(dname):
2760 try:
2760 try:
2761 for cell in get_cells():
2761 for cell in get_cells():
2762 result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
2762 result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
2763 if raise_exceptions:
2763 if raise_exceptions:
2764 result.raise_error()
2764 result.raise_error()
2765 elif not result.success:
2765 elif not result.success:
2766 break
2766 break
2767 except:
2767 except:
2768 if raise_exceptions:
2768 if raise_exceptions:
2769 raise
2769 raise
2770 self.showtraceback()
2770 self.showtraceback()
2771 warn('Unknown failure executing file: <%s>' % fname)
2771 warn('Unknown failure executing file: <%s>' % fname)
2772
2772
2773 def safe_run_module(self, mod_name, where):
2773 def safe_run_module(self, mod_name, where):
2774 """A safe version of runpy.run_module().
2774 """A safe version of runpy.run_module().
2775
2775
2776 This version will never throw an exception, but instead print
2776 This version will never throw an exception, but instead print
2777 helpful error messages to the screen.
2777 helpful error messages to the screen.
2778
2778
2779 `SystemExit` exceptions with status code 0 or None are ignored.
2779 `SystemExit` exceptions with status code 0 or None are ignored.
2780
2780
2781 Parameters
2781 Parameters
2782 ----------
2782 ----------
2783 mod_name : string
2783 mod_name : string
2784 The name of the module to be executed.
2784 The name of the module to be executed.
2785 where : dict
2785 where : dict
2786 The globals namespace.
2786 The globals namespace.
2787 """
2787 """
2788 try:
2788 try:
2789 try:
2789 try:
2790 where.update(
2790 where.update(
2791 runpy.run_module(str(mod_name), run_name="__main__",
2791 runpy.run_module(str(mod_name), run_name="__main__",
2792 alter_sys=True)
2792 alter_sys=True)
2793 )
2793 )
2794 except SystemExit as status:
2794 except SystemExit as status:
2795 if status.code:
2795 if status.code:
2796 raise
2796 raise
2797 except:
2797 except:
2798 self.showtraceback()
2798 self.showtraceback()
2799 warn('Unknown failure executing module: <%s>' % mod_name)
2799 warn('Unknown failure executing module: <%s>' % mod_name)
2800
2800
2801 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2801 def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
2802 """Run a complete IPython cell.
2802 """Run a complete IPython cell.
2803
2803
2804 Parameters
2804 Parameters
2805 ----------
2805 ----------
2806 raw_cell : str
2806 raw_cell : str
2807 The code (including IPython code such as %magic functions) to run.
2807 The code (including IPython code such as %magic functions) to run.
2808 store_history : bool
2808 store_history : bool
2809 If True, the raw and translated cell will be stored in IPython's
2809 If True, the raw and translated cell will be stored in IPython's
2810 history. For user code calling back into IPython's machinery, this
2810 history. For user code calling back into IPython's machinery, this
2811 should be set to False.
2811 should be set to False.
2812 silent : bool
2812 silent : bool
2813 If True, avoid side-effects, such as implicit displayhooks and
2813 If True, avoid side-effects, such as implicit displayhooks and
2814 and logging. silent=True forces store_history=False.
2814 and logging. silent=True forces store_history=False.
2815 shell_futures : bool
2815 shell_futures : bool
2816 If True, the code will share future statements with the interactive
2816 If True, the code will share future statements with the interactive
2817 shell. It will both be affected by previous __future__ imports, and
2817 shell. It will both be affected by previous __future__ imports, and
2818 any __future__ imports in the code will affect the shell. If False,
2818 any __future__ imports in the code will affect the shell. If False,
2819 __future__ imports are not shared in either direction.
2819 __future__ imports are not shared in either direction.
2820
2820
2821 Returns
2821 Returns
2822 -------
2822 -------
2823 result : :class:`ExecutionResult`
2823 result : :class:`ExecutionResult`
2824 """
2824 """
2825 result = None
2825 result = None
2826 try:
2826 try:
2827 result = self._run_cell(
2827 result = self._run_cell(
2828 raw_cell, store_history, silent, shell_futures)
2828 raw_cell, store_history, silent, shell_futures)
2829 finally:
2829 finally:
2830 self.events.trigger('post_execute')
2830 self.events.trigger('post_execute')
2831 if not silent:
2831 if not silent:
2832 self.events.trigger('post_run_cell', result)
2832 self.events.trigger('post_run_cell', result)
2833 return result
2833 return result
2834
2834
2835 def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
2835 def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
2836 """Internal method to run a complete IPython cell."""
2836 """Internal method to run a complete IPython cell."""
2837 coro = self.run_cell_async(
2837 coro = self.run_cell_async(
2838 raw_cell,
2838 raw_cell,
2839 store_history=store_history,
2839 store_history=store_history,
2840 silent=silent,
2840 silent=silent,
2841 shell_futures=shell_futures,
2841 shell_futures=shell_futures,
2842 )
2842 )
2843
2843
2844 # run_cell_async is async, but may not actually need an eventloop.
2844 # run_cell_async is async, but may not actually need an eventloop.
2845 # when this is the case, we want to run it using the pseudo_sync_runner
2845 # when this is the case, we want to run it using the pseudo_sync_runner
2846 # so that code can invoke eventloops (for example via the %run , and
2846 # so that code can invoke eventloops (for example via the %run , and
2847 # `%paste` magic.
2847 # `%paste` magic.
2848 if self.should_run_async(raw_cell):
2848 if self.should_run_async(raw_cell):
2849 runner = self.loop_runner
2849 runner = self.loop_runner
2850 else:
2850 else:
2851 runner = _pseudo_sync_runner
2851 runner = _pseudo_sync_runner
2852
2852
2853 try:
2853 try:
2854 return runner(coro)
2854 return runner(coro)
2855 except BaseException as e:
2855 except BaseException as e:
2856 info = ExecutionInfo(raw_cell, store_history, silent, shell_futures)
2856 info = ExecutionInfo(raw_cell, store_history, silent, shell_futures)
2857 result = ExecutionResult(info)
2857 result = ExecutionResult(info)
2858 result.error_in_exec = e
2858 result.error_in_exec = e
2859 self.showtraceback(running_compiled_code=True)
2859 self.showtraceback(running_compiled_code=True)
2860 return result
2860 return result
2861 return
2861 return
2862
2862
2863 def should_run_async(self, raw_cell: str) -> bool:
2863 def should_run_async(self, raw_cell: str) -> bool:
2864 """Return whether a cell should be run asynchronously via a coroutine runner
2864 """Return whether a cell should be run asynchronously via a coroutine runner
2865
2865
2866 Parameters
2866 Parameters
2867 ----------
2867 ----------
2868 raw_cell: str
2868 raw_cell: str
2869 The code to be executed
2869 The code to be executed
2870
2870
2871 Returns
2871 Returns
2872 -------
2872 -------
2873 result: bool
2873 result: bool
2874 Whether the code needs to be run with a coroutine runner or not
2874 Whether the code needs to be run with a coroutine runner or not
2875
2875
2876 .. versionadded: 7.0
2876 .. versionadded: 7.0
2877 """
2877 """
2878 if not self.autoawait:
2878 if not self.autoawait:
2879 return False
2879 return False
2880 try:
2880 try:
2881 cell = self.transform_cell(raw_cell)
2881 cell = self.transform_cell(raw_cell)
2882 except Exception:
2882 except Exception:
2883 # any exception during transform will be raised
2883 # any exception during transform will be raised
2884 # prior to execution
2884 # prior to execution
2885 return False
2885 return False
2886 return _should_be_async(cell)
2886 return _should_be_async(cell)
2887
2887
2888 @asyncio.coroutine
2888 @asyncio.coroutine
2889 def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
2889 def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
2890 """Run a complete IPython cell asynchronously.
2890 """Run a complete IPython cell asynchronously.
2891
2891
2892 Parameters
2892 Parameters
2893 ----------
2893 ----------
2894 raw_cell : str
2894 raw_cell : str
2895 The code (including IPython code such as %magic functions) to run.
2895 The code (including IPython code such as %magic functions) to run.
2896 store_history : bool
2896 store_history : bool
2897 If True, the raw and translated cell will be stored in IPython's
2897 If True, the raw and translated cell will be stored in IPython's
2898 history. For user code calling back into IPython's machinery, this
2898 history. For user code calling back into IPython's machinery, this
2899 should be set to False.
2899 should be set to False.
2900 silent : bool
2900 silent : bool
2901 If True, avoid side-effects, such as implicit displayhooks and
2901 If True, avoid side-effects, such as implicit displayhooks and
2902 and logging. silent=True forces store_history=False.
2902 and logging. silent=True forces store_history=False.
2903 shell_futures : bool
2903 shell_futures : bool
2904 If True, the code will share future statements with the interactive
2904 If True, the code will share future statements with the interactive
2905 shell. It will both be affected by previous __future__ imports, and
2905 shell. It will both be affected by previous __future__ imports, and
2906 any __future__ imports in the code will affect the shell. If False,
2906 any __future__ imports in the code will affect the shell. If False,
2907 __future__ imports are not shared in either direction.
2907 __future__ imports are not shared in either direction.
2908
2908
2909 Returns
2909 Returns
2910 -------
2910 -------
2911 result : :class:`ExecutionResult`
2911 result : :class:`ExecutionResult`
2912
2912
2913 .. versionadded: 7.0
2913 .. versionadded: 7.0
2914 """
2914 """
2915 info = ExecutionInfo(
2915 info = ExecutionInfo(
2916 raw_cell, store_history, silent, shell_futures)
2916 raw_cell, store_history, silent, shell_futures)
2917 result = ExecutionResult(info)
2917 result = ExecutionResult(info)
2918
2918
2919 if (not raw_cell) or raw_cell.isspace():
2919 if (not raw_cell) or raw_cell.isspace():
2920 self.last_execution_succeeded = True
2920 self.last_execution_succeeded = True
2921 self.last_execution_result = result
2921 self.last_execution_result = result
2922 return result
2922 return result
2923
2923
2924 if silent:
2924 if silent:
2925 store_history = False
2925 store_history = False
2926
2926
2927 if store_history:
2927 if store_history:
2928 result.execution_count = self.execution_count
2928 result.execution_count = self.execution_count
2929
2929
2930 def error_before_exec(value):
2930 def error_before_exec(value):
2931 if store_history:
2931 if store_history:
2932 self.execution_count += 1
2932 self.execution_count += 1
2933 result.error_before_exec = value
2933 result.error_before_exec = value
2934 self.last_execution_succeeded = False
2934 self.last_execution_succeeded = False
2935 self.last_execution_result = result
2935 self.last_execution_result = result
2936 return result
2936 return result
2937
2937
2938 self.events.trigger('pre_execute')
2938 self.events.trigger('pre_execute')
2939 if not silent:
2939 if not silent:
2940 self.events.trigger('pre_run_cell', info)
2940 self.events.trigger('pre_run_cell', info)
2941
2941
2942 # If any of our input transformation (input_transformer_manager or
2942 # If any of our input transformation (input_transformer_manager or
2943 # prefilter_manager) raises an exception, we store it in this variable
2943 # prefilter_manager) raises an exception, we store it in this variable
2944 # so that we can display the error after logging the input and storing
2944 # so that we can display the error after logging the input and storing
2945 # it in the history.
2945 # it in the history.
2946 try:
2946 try:
2947 cell = self.transform_cell(raw_cell)
2947 cell = self.transform_cell(raw_cell)
2948 except Exception:
2948 except Exception:
2949 preprocessing_exc_tuple = sys.exc_info()
2949 preprocessing_exc_tuple = sys.exc_info()
2950 cell = raw_cell # cell has to exist so it can be stored/logged
2950 cell = raw_cell # cell has to exist so it can be stored/logged
2951 else:
2951 else:
2952 preprocessing_exc_tuple = None
2952 preprocessing_exc_tuple = None
2953
2953
2954 # Store raw and processed history
2954 # Store raw and processed history
2955 if store_history:
2955 if store_history:
2956 self.history_manager.store_inputs(self.execution_count,
2956 self.history_manager.store_inputs(self.execution_count,
2957 cell, raw_cell)
2957 cell, raw_cell)
2958 if not silent:
2958 if not silent:
2959 self.logger.log(cell, raw_cell)
2959 self.logger.log(cell, raw_cell)
2960
2960
2961 # Display the exception if input processing failed.
2961 # Display the exception if input processing failed.
2962 if preprocessing_exc_tuple is not None:
2962 if preprocessing_exc_tuple is not None:
2963 self.showtraceback(preprocessing_exc_tuple)
2963 self.showtraceback(preprocessing_exc_tuple)
2964 if store_history:
2964 if store_history:
2965 self.execution_count += 1
2965 self.execution_count += 1
2966 return error_before_exec(preprocessing_exc_tuple[2])
2966 return error_before_exec(preprocessing_exc_tuple[2])
2967
2967
2968 # Our own compiler remembers the __future__ environment. If we want to
2968 # Our own compiler remembers the __future__ environment. If we want to
2969 # run code with a separate __future__ environment, use the default
2969 # run code with a separate __future__ environment, use the default
2970 # compiler
2970 # compiler
2971 compiler = self.compile if shell_futures else CachingCompiler()
2971 compiler = self.compile if shell_futures else CachingCompiler()
2972
2972
2973 _run_async = False
2973 _run_async = False
2974
2974
2975 with self.builtin_trap:
2975 with self.builtin_trap:
2976 cell_name = self.compile.cache(cell, self.execution_count)
2976 cell_name = self.compile.cache(cell, self.execution_count)
2977
2977
2978 with self.display_trap:
2978 with self.display_trap:
2979 # Compile to bytecode
2979 # Compile to bytecode
2980 try:
2980 try:
2981 if self.autoawait and _should_be_async(cell):
2981 if self.autoawait and _should_be_async(cell):
2982 # the code AST below will not be user code: we wrap it
2982 # the code AST below will not be user code: we wrap it
2983 # in an `async def`. This will likely make some AST
2983 # in an `async def`. This will likely make some AST
2984 # transformer below miss some transform opportunity and
2984 # transformer below miss some transform opportunity and
2985 # introduce a small coupling to run_code (in which we
2985 # introduce a small coupling to run_code (in which we
2986 # bake some assumptions of what _ast_asyncify returns.
2986 # bake some assumptions of what _ast_asyncify returns.
2987 # they are ways around (like grafting part of the ast
2987 # they are ways around (like grafting part of the ast
2988 # later:
2988 # later:
2989 # - Here, return code_ast.body[0].body[1:-1], as well
2989 # - Here, return code_ast.body[0].body[1:-1], as well
2990 # as last expression in return statement which is
2990 # as last expression in return statement which is
2991 # the user code part.
2991 # the user code part.
2992 # - Let it go through the AST transformers, and graft
2992 # - Let it go through the AST transformers, and graft
2993 # - it back after the AST transform
2993 # - it back after the AST transform
2994 # But that seem unreasonable, at least while we
2994 # But that seem unreasonable, at least while we
2995 # do not need it.
2995 # do not need it.
2996 code_ast = _ast_asyncify(cell, 'async-def-wrapper')
2996 code_ast = _ast_asyncify(cell, 'async-def-wrapper')
2997 _run_async = True
2997 _run_async = True
2998 else:
2998 else:
2999 code_ast = compiler.ast_parse(cell, filename=cell_name)
2999 code_ast = compiler.ast_parse(cell, filename=cell_name)
3000 except self.custom_exceptions as e:
3000 except self.custom_exceptions as e:
3001 etype, value, tb = sys.exc_info()
3001 etype, value, tb = sys.exc_info()
3002 self.CustomTB(etype, value, tb)
3002 self.CustomTB(etype, value, tb)
3003 return error_before_exec(e)
3003 return error_before_exec(e)
3004 except IndentationError as e:
3004 except IndentationError as e:
3005 self.showindentationerror()
3005 self.showindentationerror()
3006 return error_before_exec(e)
3006 return error_before_exec(e)
3007 except (OverflowError, SyntaxError, ValueError, TypeError,
3007 except (OverflowError, SyntaxError, ValueError, TypeError,
3008 MemoryError) as e:
3008 MemoryError) as e:
3009 self.showsyntaxerror()
3009 self.showsyntaxerror()
3010 return error_before_exec(e)
3010 return error_before_exec(e)
3011
3011
3012 # Apply AST transformations
3012 # Apply AST transformations
3013 try:
3013 try:
3014 code_ast = self.transform_ast(code_ast)
3014 code_ast = self.transform_ast(code_ast)
3015 except InputRejected as e:
3015 except InputRejected as e:
3016 self.showtraceback()
3016 self.showtraceback()
3017 return error_before_exec(e)
3017 return error_before_exec(e)
3018
3018
3019 # Give the displayhook a reference to our ExecutionResult so it
3019 # Give the displayhook a reference to our ExecutionResult so it
3020 # can fill in the output value.
3020 # can fill in the output value.
3021 self.displayhook.exec_result = result
3021 self.displayhook.exec_result = result
3022
3022
3023 # Execute the user code
3023 # Execute the user code
3024 interactivity = "none" if silent else self.ast_node_interactivity
3024 interactivity = "none" if silent else self.ast_node_interactivity
3025 if _run_async:
3025 if _run_async:
3026 interactivity = 'async'
3026 interactivity = 'async'
3027
3027
3028 has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
3028 has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
3029 interactivity=interactivity, compiler=compiler, result=result)
3029 interactivity=interactivity, compiler=compiler, result=result)
3030
3030
3031 self.last_execution_succeeded = not has_raised
3031 self.last_execution_succeeded = not has_raised
3032 self.last_execution_result = result
3032 self.last_execution_result = result
3033
3033
3034 # Reset this so later displayed values do not modify the
3034 # Reset this so later displayed values do not modify the
3035 # ExecutionResult
3035 # ExecutionResult
3036 self.displayhook.exec_result = None
3036 self.displayhook.exec_result = None
3037
3037
3038 if store_history:
3038 if store_history:
3039 # Write output to the database. Does nothing unless
3039 # Write output to the database. Does nothing unless
3040 # history output logging is enabled.
3040 # history output logging is enabled.
3041 self.history_manager.store_output(self.execution_count)
3041 self.history_manager.store_output(self.execution_count)
3042 # Each cell is a *single* input, regardless of how many lines it has
3042 # Each cell is a *single* input, regardless of how many lines it has
3043 self.execution_count += 1
3043 self.execution_count += 1
3044
3044
3045 return result
3045 return result
3046
3046
3047 def transform_cell(self, raw_cell):
3047 def transform_cell(self, raw_cell):
3048 """Transform an input cell before parsing it.
3048 """Transform an input cell before parsing it.
3049
3049
3050 Static transformations, implemented in IPython.core.inputtransformer2,
3050 Static transformations, implemented in IPython.core.inputtransformer2,
3051 deal with things like ``%magic`` and ``!system`` commands.
3051 deal with things like ``%magic`` and ``!system`` commands.
3052 These run on all input.
3052 These run on all input.
3053 Dynamic transformations, for things like unescaped magics and the exit
3053 Dynamic transformations, for things like unescaped magics and the exit
3054 autocall, depend on the state of the interpreter.
3054 autocall, depend on the state of the interpreter.
3055 These only apply to single line inputs.
3055 These only apply to single line inputs.
3056
3056
3057 These string-based transformations are followed by AST transformations;
3057 These string-based transformations are followed by AST transformations;
3058 see :meth:`transform_ast`.
3058 see :meth:`transform_ast`.
3059 """
3059 """
3060 # Static input transformations
3060 # Static input transformations
3061 cell = self.input_transformer_manager.transform_cell(raw_cell)
3061 cell = self.input_transformer_manager.transform_cell(raw_cell)
3062
3062
3063 if len(cell.splitlines()) == 1:
3063 if len(cell.splitlines()) == 1:
3064 # Dynamic transformations - only applied for single line commands
3064 # Dynamic transformations - only applied for single line commands
3065 with self.builtin_trap:
3065 with self.builtin_trap:
3066 # use prefilter_lines to handle trailing newlines
3066 # use prefilter_lines to handle trailing newlines
3067 # restore trailing newline for ast.parse
3067 # restore trailing newline for ast.parse
3068 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
3068 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
3069
3069
3070 lines = cell.splitlines(keepends=True)
3070 lines = cell.splitlines(keepends=True)
3071 for transform in self.input_transformers_post:
3071 for transform in self.input_transformers_post:
3072 lines = transform(lines)
3072 lines = transform(lines)
3073 cell = ''.join(lines)
3073 cell = ''.join(lines)
3074
3074
3075 return cell
3075 return cell
3076
3076
3077 def transform_ast(self, node):
3077 def transform_ast(self, node):
3078 """Apply the AST transformations from self.ast_transformers
3078 """Apply the AST transformations from self.ast_transformers
3079
3079
3080 Parameters
3080 Parameters
3081 ----------
3081 ----------
3082 node : ast.Node
3082 node : ast.Node
3083 The root node to be transformed. Typically called with the ast.Module
3083 The root node to be transformed. Typically called with the ast.Module
3084 produced by parsing user input.
3084 produced by parsing user input.
3085
3085
3086 Returns
3086 Returns
3087 -------
3087 -------
3088 An ast.Node corresponding to the node it was called with. Note that it
3088 An ast.Node corresponding to the node it was called with. Note that it
3089 may also modify the passed object, so don't rely on references to the
3089 may also modify the passed object, so don't rely on references to the
3090 original AST.
3090 original AST.
3091 """
3091 """
3092 for transformer in self.ast_transformers:
3092 for transformer in self.ast_transformers:
3093 try:
3093 try:
3094 node = transformer.visit(node)
3094 node = transformer.visit(node)
3095 except InputRejected:
3095 except InputRejected:
3096 # User-supplied AST transformers can reject an input by raising
3096 # User-supplied AST transformers can reject an input by raising
3097 # an InputRejected. Short-circuit in this case so that we
3097 # an InputRejected. Short-circuit in this case so that we
3098 # don't unregister the transform.
3098 # don't unregister the transform.
3099 raise
3099 raise
3100 except Exception:
3100 except Exception:
3101 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
3101 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
3102 self.ast_transformers.remove(transformer)
3102 self.ast_transformers.remove(transformer)
3103
3103
3104 if self.ast_transformers:
3104 if self.ast_transformers:
3105 ast.fix_missing_locations(node)
3105 ast.fix_missing_locations(node)
3106 return node
3106 return node
3107
3107
3108 @asyncio.coroutine
3108 @asyncio.coroutine
3109 def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
3109 def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
3110 compiler=compile, result=None):
3110 compiler=compile, result=None):
3111 """Run a sequence of AST nodes. The execution mode depends on the
3111 """Run a sequence of AST nodes. The execution mode depends on the
3112 interactivity parameter.
3112 interactivity parameter.
3113
3113
3114 Parameters
3114 Parameters
3115 ----------
3115 ----------
3116 nodelist : list
3116 nodelist : list
3117 A sequence of AST nodes to run.
3117 A sequence of AST nodes to run.
3118 cell_name : str
3118 cell_name : str
3119 Will be passed to the compiler as the filename of the cell. Typically
3119 Will be passed to the compiler as the filename of the cell. Typically
3120 the value returned by ip.compile.cache(cell).
3120 the value returned by ip.compile.cache(cell).
3121 interactivity : str
3121 interactivity : str
3122 'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
3122 'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
3123 specifying which nodes should be run interactively (displaying output
3123 specifying which nodes should be run interactively (displaying output
3124 from expressions). 'last_expr' will run the last node interactively
3124 from expressions). 'last_expr' will run the last node interactively
3125 only if it is an expression (i.e. expressions in loops or other blocks
3125 only if it is an expression (i.e. expressions in loops or other blocks
3126 are not displayed) 'last_expr_or_assign' will run the last expression
3126 are not displayed) 'last_expr_or_assign' will run the last expression
3127 or the last assignment. Other values for this parameter will raise a
3127 or the last assignment. Other values for this parameter will raise a
3128 ValueError.
3128 ValueError.
3129
3129
3130 Experimental value: 'async' Will try to run top level interactive
3130 Experimental value: 'async' Will try to run top level interactive
3131 async/await code in default runner, this will not respect the
3131 async/await code in default runner, this will not respect the
3132 interactivty setting and will only run the last node if it is an
3132 interactivty setting and will only run the last node if it is an
3133 expression.
3133 expression.
3134
3134
3135 compiler : callable
3135 compiler : callable
3136 A function with the same interface as the built-in compile(), to turn
3136 A function with the same interface as the built-in compile(), to turn
3137 the AST nodes into code objects. Default is the built-in compile().
3137 the AST nodes into code objects. Default is the built-in compile().
3138 result : ExecutionResult, optional
3138 result : ExecutionResult, optional
3139 An object to store exceptions that occur during execution.
3139 An object to store exceptions that occur during execution.
3140
3140
3141 Returns
3141 Returns
3142 -------
3142 -------
3143 True if an exception occurred while running code, False if it finished
3143 True if an exception occurred while running code, False if it finished
3144 running.
3144 running.
3145 """
3145 """
3146 if not nodelist:
3146 if not nodelist:
3147 return
3147 return
3148 if interactivity == 'last_expr_or_assign':
3148 if interactivity == 'last_expr_or_assign':
3149 if isinstance(nodelist[-1], _assign_nodes):
3149 if isinstance(nodelist[-1], _assign_nodes):
3150 asg = nodelist[-1]
3150 asg = nodelist[-1]
3151 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
3151 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
3152 target = asg.targets[0]
3152 target = asg.targets[0]
3153 elif isinstance(asg, _single_targets_nodes):
3153 elif isinstance(asg, _single_targets_nodes):
3154 target = asg.target
3154 target = asg.target
3155 else:
3155 else:
3156 target = None
3156 target = None
3157 if isinstance(target, ast.Name):
3157 if isinstance(target, ast.Name):
3158 nnode = ast.Expr(ast.Name(target.id, ast.Load()))
3158 nnode = ast.Expr(ast.Name(target.id, ast.Load()))
3159 ast.fix_missing_locations(nnode)
3159 ast.fix_missing_locations(nnode)
3160 nodelist.append(nnode)
3160 nodelist.append(nnode)
3161 interactivity = 'last_expr'
3161 interactivity = 'last_expr'
3162
3162
3163 _async = False
3163 _async = False
3164 if interactivity == 'last_expr':
3164 if interactivity == 'last_expr':
3165 if isinstance(nodelist[-1], ast.Expr):
3165 if isinstance(nodelist[-1], ast.Expr):
3166 interactivity = "last"
3166 interactivity = "last"
3167 else:
3167 else:
3168 interactivity = "none"
3168 interactivity = "none"
3169
3169
3170 if interactivity == 'none':
3170 if interactivity == 'none':
3171 to_run_exec, to_run_interactive = nodelist, []
3171 to_run_exec, to_run_interactive = nodelist, []
3172 elif interactivity == 'last':
3172 elif interactivity == 'last':
3173 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
3173 to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
3174 elif interactivity == 'all':
3174 elif interactivity == 'all':
3175 to_run_exec, to_run_interactive = [], nodelist
3175 to_run_exec, to_run_interactive = [], nodelist
3176 elif interactivity == 'async':
3176 elif interactivity == 'async':
3177 _async = True
3177 _async = True
3178 else:
3178 else:
3179 raise ValueError("Interactivity was %r" % interactivity)
3179 raise ValueError("Interactivity was %r" % interactivity)
3180 try:
3180 try:
3181 if _async:
3181 if _async:
3182 # If interactivity is async the semantics of run_code are
3182 # If interactivity is async the semantics of run_code are
3183 # completely different Skip usual machinery.
3183 # completely different Skip usual machinery.
3184 mod = ast.Module(nodelist)
3184 mod = ast.Module(nodelist)
3185 async_wrapper_code = compiler(mod, 'cell_name', 'exec')
3185 async_wrapper_code = compiler(mod, 'cell_name', 'exec')
3186 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
3186 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
3187 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
3187 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
3188 if (yield from self.run_code(async_code, result, async_=True)):
3188 if (yield from self.run_code(async_code, result, async_=True)):
3189 return True
3189 return True
3190 else:
3190 else:
3191 for i, node in enumerate(to_run_exec):
3191 for i, node in enumerate(to_run_exec):
3192 mod = ast.Module([node])
3192 mod = ast.Module([node])
3193 code = compiler(mod, cell_name, "exec")
3193 code = compiler(mod, cell_name, "exec")
3194 if (yield from self.run_code(code, result)):
3194 if (yield from self.run_code(code, result)):
3195 return True
3195 return True
3196
3196
3197 for i, node in enumerate(to_run_interactive):
3197 for i, node in enumerate(to_run_interactive):
3198 mod = ast.Interactive([node])
3198 mod = ast.Interactive([node])
3199 code = compiler(mod, cell_name, "single")
3199 code = compiler(mod, cell_name, "single")
3200 if (yield from self.run_code(code, result)):
3200 if (yield from self.run_code(code, result)):
3201 return True
3201 return True
3202
3202
3203 # Flush softspace
3203 # Flush softspace
3204 if softspace(sys.stdout, 0):
3204 if softspace(sys.stdout, 0):
3205 print()
3205 print()
3206
3206
3207 except:
3207 except:
3208 # It's possible to have exceptions raised here, typically by
3208 # It's possible to have exceptions raised here, typically by
3209 # compilation of odd code (such as a naked 'return' outside a
3209 # compilation of odd code (such as a naked 'return' outside a
3210 # function) that did parse but isn't valid. Typically the exception
3210 # function) that did parse but isn't valid. Typically the exception
3211 # is a SyntaxError, but it's safest just to catch anything and show
3211 # is a SyntaxError, but it's safest just to catch anything and show
3212 # the user a traceback.
3212 # the user a traceback.
3213
3213
3214 # We do only one try/except outside the loop to minimize the impact
3214 # We do only one try/except outside the loop to minimize the impact
3215 # on runtime, and also because if any node in the node list is
3215 # on runtime, and also because if any node in the node list is
3216 # broken, we should stop execution completely.
3216 # broken, we should stop execution completely.
3217 if result:
3217 if result:
3218 result.error_before_exec = sys.exc_info()[1]
3218 result.error_before_exec = sys.exc_info()[1]
3219 self.showtraceback()
3219 self.showtraceback()
3220 return True
3220 return True
3221
3221
3222 return False
3222 return False
3223
3223
3224 def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
3224 def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
3225 """
3225 """
3226 Evaluate an asynchronous code object using a code runner
3226 Evaluate an asynchronous code object using a code runner
3227
3227
3228 Fake asynchronous execution of code_object in a namespace via a proxy namespace.
3228 Fake asynchronous execution of code_object in a namespace via a proxy namespace.
3229
3229
3230 Returns coroutine object, which can be executed via async loop runner
3230 Returns coroutine object, which can be executed via async loop runner
3231
3231
3232 WARNING: The semantics of `async_exec` are quite different from `exec`,
3232 WARNING: The semantics of `async_exec` are quite different from `exec`,
3233 in particular you can only pass a single namespace. It also return a
3233 in particular you can only pass a single namespace. It also return a
3234 handle to the value of the last things returned by code_object.
3234 handle to the value of the last things returned by code_object.
3235 """
3235 """
3236
3236
3237 return eval(code_obj, user_ns)
3237 return eval(code_obj, user_ns)
3238
3238
3239 @asyncio.coroutine
3239 @asyncio.coroutine
3240 def run_code(self, code_obj, result=None, *, async_=False):
3240 def run_code(self, code_obj, result=None, *, async_=False):
3241 """Execute a code object.
3241 """Execute a code object.
3242
3242
3243 When an exception occurs, self.showtraceback() is called to display a
3243 When an exception occurs, self.showtraceback() is called to display a
3244 traceback.
3244 traceback.
3245
3245
3246 Parameters
3246 Parameters
3247 ----------
3247 ----------
3248 code_obj : code object
3248 code_obj : code object
3249 A compiled code object, to be executed
3249 A compiled code object, to be executed
3250 result : ExecutionResult, optional
3250 result : ExecutionResult, optional
3251 An object to store exceptions that occur during execution.
3251 An object to store exceptions that occur during execution.
3252 async_ : Bool (Experimental)
3252 async_ : Bool (Experimental)
3253 Attempt to run top-level asynchronous code in a default loop.
3253 Attempt to run top-level asynchronous code in a default loop.
3254
3254
3255 Returns
3255 Returns
3256 -------
3256 -------
3257 False : successful execution.
3257 False : successful execution.
3258 True : an error occurred.
3258 True : an error occurred.
3259 """
3259 """
3260 # Set our own excepthook in case the user code tries to call it
3260 # Set our own excepthook in case the user code tries to call it
3261 # directly, so that the IPython crash handler doesn't get triggered
3261 # directly, so that the IPython crash handler doesn't get triggered
3262 old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
3262 old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
3263
3263
3264 # we save the original sys.excepthook in the instance, in case config
3264 # we save the original sys.excepthook in the instance, in case config
3265 # code (such as magics) needs access to it.
3265 # code (such as magics) needs access to it.
3266 self.sys_excepthook = old_excepthook
3266 self.sys_excepthook = old_excepthook
3267 outflag = True # happens in more places, so it's easier as default
3267 outflag = True # happens in more places, so it's easier as default
3268 try:
3268 try:
3269 try:
3269 try:
3270 self.hooks.pre_run_code_hook()
3270 self.hooks.pre_run_code_hook()
3271 if async_:
3271 if async_:
3272 last_expr = (yield from self._async_exec(code_obj, self.user_ns))
3272 last_expr = (yield from self._async_exec(code_obj, self.user_ns))
3273 code = compile('last_expr', 'fake', "single")
3273 code = compile('last_expr', 'fake', "single")
3274 exec(code, {'last_expr': last_expr})
3274 exec(code, {'last_expr': last_expr})
3275 else:
3275 else:
3276 exec(code_obj, self.user_global_ns, self.user_ns)
3276 exec(code_obj, self.user_global_ns, self.user_ns)
3277 finally:
3277 finally:
3278 # Reset our crash handler in place
3278 # Reset our crash handler in place
3279 sys.excepthook = old_excepthook
3279 sys.excepthook = old_excepthook
3280 except SystemExit as e:
3280 except SystemExit as e:
3281 if result is not None:
3281 if result is not None:
3282 result.error_in_exec = e
3282 result.error_in_exec = e
3283 self.showtraceback(exception_only=True)
3283 self.showtraceback(exception_only=True)
3284 warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
3284 warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
3285 except self.custom_exceptions:
3285 except self.custom_exceptions:
3286 etype, value, tb = sys.exc_info()
3286 etype, value, tb = sys.exc_info()
3287 if result is not None:
3287 if result is not None:
3288 result.error_in_exec = value
3288 result.error_in_exec = value
3289 self.CustomTB(etype, value, tb)
3289 self.CustomTB(etype, value, tb)
3290 except:
3290 except:
3291 if result is not None:
3291 if result is not None:
3292 result.error_in_exec = sys.exc_info()[1]
3292 result.error_in_exec = sys.exc_info()[1]
3293 self.showtraceback(running_compiled_code=True)
3293 self.showtraceback(running_compiled_code=True)
3294 else:
3294 else:
3295 outflag = False
3295 outflag = False
3296 return outflag
3296 return outflag
3297
3297
3298 # For backwards compatibility
3298 # For backwards compatibility
3299 runcode = run_code
3299 runcode = run_code
3300
3300
3301 def check_complete(self, code: str) -> Tuple[str, str]:
3301 def check_complete(self, code: str) -> Tuple[str, str]:
3302 """Return whether a block of code is ready to execute, or should be continued
3302 """Return whether a block of code is ready to execute, or should be continued
3303
3303
3304 Parameters
3304 Parameters
3305 ----------
3305 ----------
3306 source : string
3306 source : string
3307 Python input code, which can be multiline.
3307 Python input code, which can be multiline.
3308
3308
3309 Returns
3309 Returns
3310 -------
3310 -------
3311 status : str
3311 status : str
3312 One of 'complete', 'incomplete', or 'invalid' if source is not a
3312 One of 'complete', 'incomplete', or 'invalid' if source is not a
3313 prefix of valid code.
3313 prefix of valid code.
3314 indent : str
3314 indent : str
3315 When status is 'incomplete', this is some whitespace to insert on
3315 When status is 'incomplete', this is some whitespace to insert on
3316 the next line of the prompt.
3316 the next line of the prompt.
3317 """
3317 """
3318 status, nspaces = self.input_transformer_manager.check_complete(code)
3318 status, nspaces = self.input_transformer_manager.check_complete(code)
3319 return status, ' ' * (nspaces or 0)
3319 return status, ' ' * (nspaces or 0)
3320
3320
3321 #-------------------------------------------------------------------------
3321 #-------------------------------------------------------------------------
3322 # Things related to GUI support and pylab
3322 # Things related to GUI support and pylab
3323 #-------------------------------------------------------------------------
3323 #-------------------------------------------------------------------------
3324
3324
3325 active_eventloop = None
3325 active_eventloop = None
3326
3326
3327 def enable_gui(self, gui=None):
3327 def enable_gui(self, gui=None):
3328 raise NotImplementedError('Implement enable_gui in a subclass')
3328 raise NotImplementedError('Implement enable_gui in a subclass')
3329
3329
3330 def enable_matplotlib(self, gui=None):
3330 def enable_matplotlib(self, gui=None):
3331 """Enable interactive matplotlib and inline figure support.
3331 """Enable interactive matplotlib and inline figure support.
3332
3332
3333 This takes the following steps:
3333 This takes the following steps:
3334
3334
3335 1. select the appropriate eventloop and matplotlib backend
3335 1. select the appropriate eventloop and matplotlib backend
3336 2. set up matplotlib for interactive use with that backend
3336 2. set up matplotlib for interactive use with that backend
3337 3. configure formatters for inline figure display
3337 3. configure formatters for inline figure display
3338 4. enable the selected gui eventloop
3338 4. enable the selected gui eventloop
3339
3339
3340 Parameters
3340 Parameters
3341 ----------
3341 ----------
3342 gui : optional, string
3342 gui : optional, string
3343 If given, dictates the choice of matplotlib GUI backend to use
3343 If given, dictates the choice of matplotlib GUI backend to use
3344 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3344 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3345 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3345 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3346 matplotlib (as dictated by the matplotlib build-time options plus the
3346 matplotlib (as dictated by the matplotlib build-time options plus the
3347 user's matplotlibrc configuration file). Note that not all backends
3347 user's matplotlibrc configuration file). Note that not all backends
3348 make sense in all contexts, for example a terminal ipython can't
3348 make sense in all contexts, for example a terminal ipython can't
3349 display figures inline.
3349 display figures inline.
3350 """
3350 """
3351 from IPython.core import pylabtools as pt
3351 from IPython.core import pylabtools as pt
3352 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
3352 gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
3353
3353
3354 if gui != 'inline':
3354 if gui != 'inline':
3355 # If we have our first gui selection, store it
3355 # If we have our first gui selection, store it
3356 if self.pylab_gui_select is None:
3356 if self.pylab_gui_select is None:
3357 self.pylab_gui_select = gui
3357 self.pylab_gui_select = gui
3358 # Otherwise if they are different
3358 # Otherwise if they are different
3359 elif gui != self.pylab_gui_select:
3359 elif gui != self.pylab_gui_select:
3360 print('Warning: Cannot change to a different GUI toolkit: %s.'
3360 print('Warning: Cannot change to a different GUI toolkit: %s.'
3361 ' Using %s instead.' % (gui, self.pylab_gui_select))
3361 ' Using %s instead.' % (gui, self.pylab_gui_select))
3362 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
3362 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
3363
3363
3364 pt.activate_matplotlib(backend)
3364 pt.activate_matplotlib(backend)
3365 pt.configure_inline_support(self, backend)
3365 pt.configure_inline_support(self, backend)
3366
3366
3367 # Now we must activate the gui pylab wants to use, and fix %run to take
3367 # Now we must activate the gui pylab wants to use, and fix %run to take
3368 # plot updates into account
3368 # plot updates into account
3369 self.enable_gui(gui)
3369 self.enable_gui(gui)
3370 self.magics_manager.registry['ExecutionMagics'].default_runner = \
3370 self.magics_manager.registry['ExecutionMagics'].default_runner = \
3371 pt.mpl_runner(self.safe_execfile)
3371 pt.mpl_runner(self.safe_execfile)
3372
3372
3373 return gui, backend
3373 return gui, backend
3374
3374
3375 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
3375 def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
3376 """Activate pylab support at runtime.
3376 """Activate pylab support at runtime.
3377
3377
3378 This turns on support for matplotlib, preloads into the interactive
3378 This turns on support for matplotlib, preloads into the interactive
3379 namespace all of numpy and pylab, and configures IPython to correctly
3379 namespace all of numpy and pylab, and configures IPython to correctly
3380 interact with the GUI event loop. The GUI backend to be used can be
3380 interact with the GUI event loop. The GUI backend to be used can be
3381 optionally selected with the optional ``gui`` argument.
3381 optionally selected with the optional ``gui`` argument.
3382
3382
3383 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
3383 This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
3384
3384
3385 Parameters
3385 Parameters
3386 ----------
3386 ----------
3387 gui : optional, string
3387 gui : optional, string
3388 If given, dictates the choice of matplotlib GUI backend to use
3388 If given, dictates the choice of matplotlib GUI backend to use
3389 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3389 (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
3390 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3390 'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
3391 matplotlib (as dictated by the matplotlib build-time options plus the
3391 matplotlib (as dictated by the matplotlib build-time options plus the
3392 user's matplotlibrc configuration file). Note that not all backends
3392 user's matplotlibrc configuration file). Note that not all backends
3393 make sense in all contexts, for example a terminal ipython can't
3393 make sense in all contexts, for example a terminal ipython can't
3394 display figures inline.
3394 display figures inline.
3395 import_all : optional, bool, default: True
3395 import_all : optional, bool, default: True
3396 Whether to do `from numpy import *` and `from pylab import *`
3396 Whether to do `from numpy import *` and `from pylab import *`
3397 in addition to module imports.
3397 in addition to module imports.
3398 welcome_message : deprecated
3398 welcome_message : deprecated
3399 This argument is ignored, no welcome message will be displayed.
3399 This argument is ignored, no welcome message will be displayed.
3400 """
3400 """
3401 from IPython.core.pylabtools import import_pylab
3401 from IPython.core.pylabtools import import_pylab
3402
3402
3403 gui, backend = self.enable_matplotlib(gui)
3403 gui, backend = self.enable_matplotlib(gui)
3404
3404
3405 # We want to prevent the loading of pylab to pollute the user's
3405 # We want to prevent the loading of pylab to pollute the user's
3406 # namespace as shown by the %who* magics, so we execute the activation
3406 # namespace as shown by the %who* magics, so we execute the activation
3407 # code in an empty namespace, and we update *both* user_ns and
3407 # code in an empty namespace, and we update *both* user_ns and
3408 # user_ns_hidden with this information.
3408 # user_ns_hidden with this information.
3409 ns = {}
3409 ns = {}
3410 import_pylab(ns, import_all)
3410 import_pylab(ns, import_all)
3411 # warn about clobbered names
3411 # warn about clobbered names
3412 ignored = {"__builtins__"}
3412 ignored = {"__builtins__"}
3413 both = set(ns).intersection(self.user_ns).difference(ignored)
3413 both = set(ns).intersection(self.user_ns).difference(ignored)
3414 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
3414 clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
3415 self.user_ns.update(ns)
3415 self.user_ns.update(ns)
3416 self.user_ns_hidden.update(ns)
3416 self.user_ns_hidden.update(ns)
3417 return gui, backend, clobbered
3417 return gui, backend, clobbered
3418
3418
3419 #-------------------------------------------------------------------------
3419 #-------------------------------------------------------------------------
3420 # Utilities
3420 # Utilities
3421 #-------------------------------------------------------------------------
3421 #-------------------------------------------------------------------------
3422
3422
3423 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
3423 def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
3424 """Expand python variables in a string.
3424 """Expand python variables in a string.
3425
3425
3426 The depth argument indicates how many frames above the caller should
3426 The depth argument indicates how many frames above the caller should
3427 be walked to look for the local namespace where to expand variables.
3427 be walked to look for the local namespace where to expand variables.
3428
3428
3429 The global namespace for expansion is always the user's interactive
3429 The global namespace for expansion is always the user's interactive
3430 namespace.
3430 namespace.
3431 """
3431 """
3432 ns = self.user_ns.copy()
3432 ns = self.user_ns.copy()
3433 try:
3433 try:
3434 frame = sys._getframe(depth+1)
3434 frame = sys._getframe(depth+1)
3435 except ValueError:
3435 except ValueError:
3436 # This is thrown if there aren't that many frames on the stack,
3436 # This is thrown if there aren't that many frames on the stack,
3437 # e.g. if a script called run_line_magic() directly.
3437 # e.g. if a script called run_line_magic() directly.
3438 pass
3438 pass
3439 else:
3439 else:
3440 ns.update(frame.f_locals)
3440 ns.update(frame.f_locals)
3441
3441
3442 try:
3442 try:
3443 # We have to use .vformat() here, because 'self' is a valid and common
3443 # We have to use .vformat() here, because 'self' is a valid and common
3444 # name, and expanding **ns for .format() would make it collide with
3444 # name, and expanding **ns for .format() would make it collide with
3445 # the 'self' argument of the method.
3445 # the 'self' argument of the method.
3446 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
3446 cmd = formatter.vformat(cmd, args=[], kwargs=ns)
3447 except Exception:
3447 except Exception:
3448 # if formatter couldn't format, just let it go untransformed
3448 # if formatter couldn't format, just let it go untransformed
3449 pass
3449 pass
3450 return cmd
3450 return cmd
3451
3451
3452 def mktempfile(self, data=None, prefix='ipython_edit_'):
3452 def mktempfile(self, data=None, prefix='ipython_edit_'):
3453 """Make a new tempfile and return its filename.
3453 """Make a new tempfile and return its filename.
3454
3454
3455 This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
3455 This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
3456 but it registers the created filename internally so ipython cleans it up
3456 but it registers the created filename internally so ipython cleans it up
3457 at exit time.
3457 at exit time.
3458
3458
3459 Optional inputs:
3459 Optional inputs:
3460
3460
3461 - data(None): if data is given, it gets written out to the temp file
3461 - data(None): if data is given, it gets written out to the temp file
3462 immediately, and the file is closed again."""
3462 immediately, and the file is closed again."""
3463
3463
3464 dirname = tempfile.mkdtemp(prefix=prefix)
3464 dirname = tempfile.mkdtemp(prefix=prefix)
3465 self.tempdirs.append(dirname)
3465 self.tempdirs.append(dirname)
3466
3466
3467 handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
3467 handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
3468 os.close(handle) # On Windows, there can only be one open handle on a file
3468 os.close(handle) # On Windows, there can only be one open handle on a file
3469 self.tempfiles.append(filename)
3469 self.tempfiles.append(filename)
3470
3470
3471 if data:
3471 if data:
3472 tmp_file = open(filename,'w')
3472 with open(filename, 'w') as tmp_file:
3473 tmp_file.write(data)
3473 tmp_file.write(data)
3474 tmp_file.close()
3475 return filename
3474 return filename
3476
3475
3477 @undoc
3476 @undoc
3478 def write(self,data):
3477 def write(self,data):
3479 """DEPRECATED: Write a string to the default output"""
3478 """DEPRECATED: Write a string to the default output"""
3480 warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
3479 warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
3481 DeprecationWarning, stacklevel=2)
3480 DeprecationWarning, stacklevel=2)
3482 sys.stdout.write(data)
3481 sys.stdout.write(data)
3483
3482
3484 @undoc
3483 @undoc
3485 def write_err(self,data):
3484 def write_err(self,data):
3486 """DEPRECATED: Write a string to the default error output"""
3485 """DEPRECATED: Write a string to the default error output"""
3487 warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
3486 warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
3488 DeprecationWarning, stacklevel=2)
3487 DeprecationWarning, stacklevel=2)
3489 sys.stderr.write(data)
3488 sys.stderr.write(data)
3490
3489
3491 def ask_yes_no(self, prompt, default=None, interrupt=None):
3490 def ask_yes_no(self, prompt, default=None, interrupt=None):
3492 if self.quiet:
3491 if self.quiet:
3493 return True
3492 return True
3494 return ask_yes_no(prompt,default,interrupt)
3493 return ask_yes_no(prompt,default,interrupt)
3495
3494
3496 def show_usage(self):
3495 def show_usage(self):
3497 """Show a usage message"""
3496 """Show a usage message"""
3498 page.page(IPython.core.usage.interactive_usage)
3497 page.page(IPython.core.usage.interactive_usage)
3499
3498
3500 def extract_input_lines(self, range_str, raw=False):
3499 def extract_input_lines(self, range_str, raw=False):
3501 """Return as a string a set of input history slices.
3500 """Return as a string a set of input history slices.
3502
3501
3503 Parameters
3502 Parameters
3504 ----------
3503 ----------
3505 range_str : string
3504 range_str : string
3506 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3505 The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
3507 since this function is for use by magic functions which get their
3506 since this function is for use by magic functions which get their
3508 arguments as strings. The number before the / is the session
3507 arguments as strings. The number before the / is the session
3509 number: ~n goes n back from the current session.
3508 number: ~n goes n back from the current session.
3510
3509
3511 raw : bool, optional
3510 raw : bool, optional
3512 By default, the processed input is used. If this is true, the raw
3511 By default, the processed input is used. If this is true, the raw
3513 input history is used instead.
3512 input history is used instead.
3514
3513
3515 Notes
3514 Notes
3516 -----
3515 -----
3517
3516
3518 Slices can be described with two notations:
3517 Slices can be described with two notations:
3519
3518
3520 * ``N:M`` -> standard python form, means including items N...(M-1).
3519 * ``N:M`` -> standard python form, means including items N...(M-1).
3521 * ``N-M`` -> include items N..M (closed endpoint).
3520 * ``N-M`` -> include items N..M (closed endpoint).
3522 """
3521 """
3523 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3522 lines = self.history_manager.get_range_by_str(range_str, raw=raw)
3524 return "\n".join(x for _, _, x in lines)
3523 return "\n".join(x for _, _, x in lines)
3525
3524
3526 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
3525 def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
3527 """Get a code string from history, file, url, or a string or macro.
3526 """Get a code string from history, file, url, or a string or macro.
3528
3527
3529 This is mainly used by magic functions.
3528 This is mainly used by magic functions.
3530
3529
3531 Parameters
3530 Parameters
3532 ----------
3531 ----------
3533
3532
3534 target : str
3533 target : str
3535
3534
3536 A string specifying code to retrieve. This will be tried respectively
3535 A string specifying code to retrieve. This will be tried respectively
3537 as: ranges of input history (see %history for syntax), url,
3536 as: ranges of input history (see %history for syntax), url,
3538 corresponding .py file, filename, or an expression evaluating to a
3537 corresponding .py file, filename, or an expression evaluating to a
3539 string or Macro in the user namespace.
3538 string or Macro in the user namespace.
3540
3539
3541 raw : bool
3540 raw : bool
3542 If true (default), retrieve raw history. Has no effect on the other
3541 If true (default), retrieve raw history. Has no effect on the other
3543 retrieval mechanisms.
3542 retrieval mechanisms.
3544
3543
3545 py_only : bool (default False)
3544 py_only : bool (default False)
3546 Only try to fetch python code, do not try alternative methods to decode file
3545 Only try to fetch python code, do not try alternative methods to decode file
3547 if unicode fails.
3546 if unicode fails.
3548
3547
3549 Returns
3548 Returns
3550 -------
3549 -------
3551 A string of code.
3550 A string of code.
3552
3551
3553 ValueError is raised if nothing is found, and TypeError if it evaluates
3552 ValueError is raised if nothing is found, and TypeError if it evaluates
3554 to an object of another type. In each case, .args[0] is a printable
3553 to an object of another type. In each case, .args[0] is a printable
3555 message.
3554 message.
3556 """
3555 """
3557 code = self.extract_input_lines(target, raw=raw) # Grab history
3556 code = self.extract_input_lines(target, raw=raw) # Grab history
3558 if code:
3557 if code:
3559 return code
3558 return code
3560 try:
3559 try:
3561 if target.startswith(('http://', 'https://')):
3560 if target.startswith(('http://', 'https://')):
3562 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
3561 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
3563 except UnicodeDecodeError:
3562 except UnicodeDecodeError:
3564 if not py_only :
3563 if not py_only :
3565 # Deferred import
3564 # Deferred import
3566 from urllib.request import urlopen
3565 from urllib.request import urlopen
3567 response = urlopen(target)
3566 response = urlopen(target)
3568 return response.read().decode('latin1')
3567 return response.read().decode('latin1')
3569 raise ValueError(("'%s' seem to be unreadable.") % target)
3568 raise ValueError(("'%s' seem to be unreadable.") % target)
3570
3569
3571 potential_target = [target]
3570 potential_target = [target]
3572 try :
3571 try :
3573 potential_target.insert(0,get_py_filename(target))
3572 potential_target.insert(0,get_py_filename(target))
3574 except IOError:
3573 except IOError:
3575 pass
3574 pass
3576
3575
3577 for tgt in potential_target :
3576 for tgt in potential_target :
3578 if os.path.isfile(tgt): # Read file
3577 if os.path.isfile(tgt): # Read file
3579 try :
3578 try :
3580 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3579 return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
3581 except UnicodeDecodeError :
3580 except UnicodeDecodeError :
3582 if not py_only :
3581 if not py_only :
3583 with io_open(tgt,'r', encoding='latin1') as f :
3582 with io_open(tgt,'r', encoding='latin1') as f :
3584 return f.read()
3583 return f.read()
3585 raise ValueError(("'%s' seem to be unreadable.") % target)
3584 raise ValueError(("'%s' seem to be unreadable.") % target)
3586 elif os.path.isdir(os.path.expanduser(tgt)):
3585 elif os.path.isdir(os.path.expanduser(tgt)):
3587 raise ValueError("'%s' is a directory, not a regular file." % target)
3586 raise ValueError("'%s' is a directory, not a regular file." % target)
3588
3587
3589 if search_ns:
3588 if search_ns:
3590 # Inspect namespace to load object source
3589 # Inspect namespace to load object source
3591 object_info = self.object_inspect(target, detail_level=1)
3590 object_info = self.object_inspect(target, detail_level=1)
3592 if object_info['found'] and object_info['source']:
3591 if object_info['found'] and object_info['source']:
3593 return object_info['source']
3592 return object_info['source']
3594
3593
3595 try: # User namespace
3594 try: # User namespace
3596 codeobj = eval(target, self.user_ns)
3595 codeobj = eval(target, self.user_ns)
3597 except Exception:
3596 except Exception:
3598 raise ValueError(("'%s' was not found in history, as a file, url, "
3597 raise ValueError(("'%s' was not found in history, as a file, url, "
3599 "nor in the user namespace.") % target)
3598 "nor in the user namespace.") % target)
3600
3599
3601 if isinstance(codeobj, str):
3600 if isinstance(codeobj, str):
3602 return codeobj
3601 return codeobj
3603 elif isinstance(codeobj, Macro):
3602 elif isinstance(codeobj, Macro):
3604 return codeobj.value
3603 return codeobj.value
3605
3604
3606 raise TypeError("%s is neither a string nor a macro." % target,
3605 raise TypeError("%s is neither a string nor a macro." % target,
3607 codeobj)
3606 codeobj)
3608
3607
3609 #-------------------------------------------------------------------------
3608 #-------------------------------------------------------------------------
3610 # Things related to IPython exiting
3609 # Things related to IPython exiting
3611 #-------------------------------------------------------------------------
3610 #-------------------------------------------------------------------------
3612 def atexit_operations(self):
3611 def atexit_operations(self):
3613 """This will be executed at the time of exit.
3612 """This will be executed at the time of exit.
3614
3613
3615 Cleanup operations and saving of persistent data that is done
3614 Cleanup operations and saving of persistent data that is done
3616 unconditionally by IPython should be performed here.
3615 unconditionally by IPython should be performed here.
3617
3616
3618 For things that may depend on startup flags or platform specifics (such
3617 For things that may depend on startup flags or platform specifics (such
3619 as having readline or not), register a separate atexit function in the
3618 as having readline or not), register a separate atexit function in the
3620 code that has the appropriate information, rather than trying to
3619 code that has the appropriate information, rather than trying to
3621 clutter
3620 clutter
3622 """
3621 """
3623 # Close the history session (this stores the end time and line count)
3622 # Close the history session (this stores the end time and line count)
3624 # this must be *before* the tempfile cleanup, in case of temporary
3623 # this must be *before* the tempfile cleanup, in case of temporary
3625 # history db
3624 # history db
3626 self.history_manager.end_session()
3625 self.history_manager.end_session()
3627
3626
3628 # Cleanup all tempfiles and folders left around
3627 # Cleanup all tempfiles and folders left around
3629 for tfile in self.tempfiles:
3628 for tfile in self.tempfiles:
3630 try:
3629 try:
3631 os.unlink(tfile)
3630 os.unlink(tfile)
3632 except OSError:
3631 except OSError:
3633 pass
3632 pass
3634
3633
3635 for tdir in self.tempdirs:
3634 for tdir in self.tempdirs:
3636 try:
3635 try:
3637 os.rmdir(tdir)
3636 os.rmdir(tdir)
3638 except OSError:
3637 except OSError:
3639 pass
3638 pass
3640
3639
3641 # Clear all user namespaces to release all references cleanly.
3640 # Clear all user namespaces to release all references cleanly.
3642 self.reset(new_session=False)
3641 self.reset(new_session=False)
3643
3642
3644 # Run user hooks
3643 # Run user hooks
3645 self.hooks.shutdown_hook()
3644 self.hooks.shutdown_hook()
3646
3645
3647 def cleanup(self):
3646 def cleanup(self):
3648 self.restore_sys_module_state()
3647 self.restore_sys_module_state()
3649
3648
3650
3649
3651 # Overridden in terminal subclass to change prompts
3650 # Overridden in terminal subclass to change prompts
3652 def switch_doctest_mode(self, mode):
3651 def switch_doctest_mode(self, mode):
3653 pass
3652 pass
3654
3653
3655
3654
3656 class InteractiveShellABC(metaclass=abc.ABCMeta):
3655 class InteractiveShellABC(metaclass=abc.ABCMeta):
3657 """An abstract base class for InteractiveShell."""
3656 """An abstract base class for InteractiveShell."""
3658
3657
3659 InteractiveShellABC.register(InteractiveShell)
3658 InteractiveShellABC.register(InteractiveShell)
@@ -1,731 +1,732 b''
1 """Implementation of code management magic functions.
1 """Implementation of code management magic functions.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2012 The IPython Development Team.
4 # Copyright (c) 2012 The IPython Development Team.
5 #
5 #
6 # Distributed under the terms of the Modified BSD License.
6 # Distributed under the terms of the Modified BSD License.
7 #
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 # Stdlib
15 # Stdlib
16 import inspect
16 import inspect
17 import io
17 import io
18 import os
18 import os
19 import re
19 import re
20 import sys
20 import sys
21 import ast
21 import ast
22 from itertools import chain
22 from itertools import chain
23 from urllib.request import urlopen
23 from urllib.request import urlopen
24 from urllib.parse import urlencode
24 from urllib.parse import urlencode
25
25
26 # Our own packages
26 # Our own packages
27 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
27 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
28 from IPython.core.macro import Macro
28 from IPython.core.macro import Macro
29 from IPython.core.magic import Magics, magics_class, line_magic
29 from IPython.core.magic import Magics, magics_class, line_magic
30 from IPython.core.oinspect import find_file, find_source_lines
30 from IPython.core.oinspect import find_file, find_source_lines
31 from IPython.testing.skipdoctest import skip_doctest
31 from IPython.testing.skipdoctest import skip_doctest
32 from IPython.utils import py3compat
32 from IPython.utils import py3compat
33 from IPython.utils.contexts import preserve_keys
33 from IPython.utils.contexts import preserve_keys
34 from IPython.utils.path import get_py_filename
34 from IPython.utils.path import get_py_filename
35 from warnings import warn
35 from warnings import warn
36 from logging import error
36 from logging import error
37 from IPython.utils.text import get_text_list
37 from IPython.utils.text import get_text_list
38
38
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40 # Magic implementation classes
40 # Magic implementation classes
41 #-----------------------------------------------------------------------------
41 #-----------------------------------------------------------------------------
42
42
43 # Used for exception handling in magic_edit
43 # Used for exception handling in magic_edit
44 class MacroToEdit(ValueError): pass
44 class MacroToEdit(ValueError): pass
45
45
46 ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
46 ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
47
47
48 # To match, e.g. 8-10 1:5 :10 3-
48 # To match, e.g. 8-10 1:5 :10 3-
49 range_re = re.compile(r"""
49 range_re = re.compile(r"""
50 (?P<start>\d+)?
50 (?P<start>\d+)?
51 ((?P<sep>[\-:])
51 ((?P<sep>[\-:])
52 (?P<end>\d+)?)?
52 (?P<end>\d+)?)?
53 $""", re.VERBOSE)
53 $""", re.VERBOSE)
54
54
55
55
56 def extract_code_ranges(ranges_str):
56 def extract_code_ranges(ranges_str):
57 """Turn a string of range for %%load into 2-tuples of (start, stop)
57 """Turn a string of range for %%load into 2-tuples of (start, stop)
58 ready to use as a slice of the content split by lines.
58 ready to use as a slice of the content split by lines.
59
59
60 Examples
60 Examples
61 --------
61 --------
62 list(extract_input_ranges("5-10 2"))
62 list(extract_input_ranges("5-10 2"))
63 [(4, 10), (1, 2)]
63 [(4, 10), (1, 2)]
64 """
64 """
65 for range_str in ranges_str.split():
65 for range_str in ranges_str.split():
66 rmatch = range_re.match(range_str)
66 rmatch = range_re.match(range_str)
67 if not rmatch:
67 if not rmatch:
68 continue
68 continue
69 sep = rmatch.group("sep")
69 sep = rmatch.group("sep")
70 start = rmatch.group("start")
70 start = rmatch.group("start")
71 end = rmatch.group("end")
71 end = rmatch.group("end")
72
72
73 if sep == '-':
73 if sep == '-':
74 start = int(start) - 1 if start else None
74 start = int(start) - 1 if start else None
75 end = int(end) if end else None
75 end = int(end) if end else None
76 elif sep == ':':
76 elif sep == ':':
77 start = int(start) - 1 if start else None
77 start = int(start) - 1 if start else None
78 end = int(end) - 1 if end else None
78 end = int(end) - 1 if end else None
79 else:
79 else:
80 end = int(start)
80 end = int(start)
81 start = int(start) - 1
81 start = int(start) - 1
82 yield (start, end)
82 yield (start, end)
83
83
84
84
85 def extract_symbols(code, symbols):
85 def extract_symbols(code, symbols):
86 """
86 """
87 Return a tuple (blocks, not_found)
87 Return a tuple (blocks, not_found)
88 where ``blocks`` is a list of code fragments
88 where ``blocks`` is a list of code fragments
89 for each symbol parsed from code, and ``not_found`` are
89 for each symbol parsed from code, and ``not_found`` are
90 symbols not found in the code.
90 symbols not found in the code.
91
91
92 For example::
92 For example::
93
93
94 In [1]: code = '''a = 10
94 In [1]: code = '''a = 10
95 ...: def b(): return 42
95 ...: def b(): return 42
96 ...: class A: pass'''
96 ...: class A: pass'''
97
97
98 In [2]: extract_symbols(code, 'A,b,z')
98 In [2]: extract_symbols(code, 'A,b,z')
99 Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z'])
99 Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z'])
100 """
100 """
101 symbols = symbols.split(',')
101 symbols = symbols.split(',')
102
102
103 # this will raise SyntaxError if code isn't valid Python
103 # this will raise SyntaxError if code isn't valid Python
104 py_code = ast.parse(code)
104 py_code = ast.parse(code)
105
105
106 marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body]
106 marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body]
107 code = code.split('\n')
107 code = code.split('\n')
108
108
109 symbols_lines = {}
109 symbols_lines = {}
110
110
111 # we already know the start_lineno of each symbol (marks).
111 # we already know the start_lineno of each symbol (marks).
112 # To find each end_lineno, we traverse in reverse order until each
112 # To find each end_lineno, we traverse in reverse order until each
113 # non-blank line
113 # non-blank line
114 end = len(code)
114 end = len(code)
115 for name, start in reversed(marks):
115 for name, start in reversed(marks):
116 while not code[end - 1].strip():
116 while not code[end - 1].strip():
117 end -= 1
117 end -= 1
118 if name:
118 if name:
119 symbols_lines[name] = (start - 1, end)
119 symbols_lines[name] = (start - 1, end)
120 end = start - 1
120 end = start - 1
121
121
122 # Now symbols_lines is a map
122 # Now symbols_lines is a map
123 # {'symbol_name': (start_lineno, end_lineno), ...}
123 # {'symbol_name': (start_lineno, end_lineno), ...}
124
124
125 # fill a list with chunks of codes for each requested symbol
125 # fill a list with chunks of codes for each requested symbol
126 blocks = []
126 blocks = []
127 not_found = []
127 not_found = []
128 for symbol in symbols:
128 for symbol in symbols:
129 if symbol in symbols_lines:
129 if symbol in symbols_lines:
130 start, end = symbols_lines[symbol]
130 start, end = symbols_lines[symbol]
131 blocks.append('\n'.join(code[start:end]) + '\n')
131 blocks.append('\n'.join(code[start:end]) + '\n')
132 else:
132 else:
133 not_found.append(symbol)
133 not_found.append(symbol)
134
134
135 return blocks, not_found
135 return blocks, not_found
136
136
137 def strip_initial_indent(lines):
137 def strip_initial_indent(lines):
138 """For %load, strip indent from lines until finding an unindented line.
138 """For %load, strip indent from lines until finding an unindented line.
139
139
140 https://github.com/ipython/ipython/issues/9775
140 https://github.com/ipython/ipython/issues/9775
141 """
141 """
142 indent_re = re.compile(r'\s+')
142 indent_re = re.compile(r'\s+')
143
143
144 it = iter(lines)
144 it = iter(lines)
145 first_line = next(it)
145 first_line = next(it)
146 indent_match = indent_re.match(first_line)
146 indent_match = indent_re.match(first_line)
147
147
148 if indent_match:
148 if indent_match:
149 # First line was indented
149 # First line was indented
150 indent = indent_match.group()
150 indent = indent_match.group()
151 yield first_line[len(indent):]
151 yield first_line[len(indent):]
152
152
153 for line in it:
153 for line in it:
154 if line.startswith(indent):
154 if line.startswith(indent):
155 yield line[len(indent):]
155 yield line[len(indent):]
156 else:
156 else:
157 # Less indented than the first line - stop dedenting
157 # Less indented than the first line - stop dedenting
158 yield line
158 yield line
159 break
159 break
160 else:
160 else:
161 yield first_line
161 yield first_line
162
162
163 # Pass the remaining lines through without dedenting
163 # Pass the remaining lines through without dedenting
164 for line in it:
164 for line in it:
165 yield line
165 yield line
166
166
167
167
168 class InteractivelyDefined(Exception):
168 class InteractivelyDefined(Exception):
169 """Exception for interactively defined variable in magic_edit"""
169 """Exception for interactively defined variable in magic_edit"""
170 def __init__(self, index):
170 def __init__(self, index):
171 self.index = index
171 self.index = index
172
172
173
173
174 @magics_class
174 @magics_class
175 class CodeMagics(Magics):
175 class CodeMagics(Magics):
176 """Magics related to code management (loading, saving, editing, ...)."""
176 """Magics related to code management (loading, saving, editing, ...)."""
177
177
178 def __init__(self, *args, **kwargs):
178 def __init__(self, *args, **kwargs):
179 self._knowntemps = set()
179 self._knowntemps = set()
180 super(CodeMagics, self).__init__(*args, **kwargs)
180 super(CodeMagics, self).__init__(*args, **kwargs)
181
181
182 @line_magic
182 @line_magic
183 def save(self, parameter_s=''):
183 def save(self, parameter_s=''):
184 """Save a set of lines or a macro to a given filename.
184 """Save a set of lines or a macro to a given filename.
185
185
186 Usage:\\
186 Usage:\\
187 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
187 %save [options] filename n1-n2 n3-n4 ... n5 .. n6 ...
188
188
189 Options:
189 Options:
190
190
191 -r: use 'raw' input. By default, the 'processed' history is used,
191 -r: use 'raw' input. By default, the 'processed' history is used,
192 so that magics are loaded in their transformed version to valid
192 so that magics are loaded in their transformed version to valid
193 Python. If this option is given, the raw input as typed as the
193 Python. If this option is given, the raw input as typed as the
194 command line is used instead.
194 command line is used instead.
195
195
196 -f: force overwrite. If file exists, %save will prompt for overwrite
196 -f: force overwrite. If file exists, %save will prompt for overwrite
197 unless -f is given.
197 unless -f is given.
198
198
199 -a: append to the file instead of overwriting it.
199 -a: append to the file instead of overwriting it.
200
200
201 This function uses the same syntax as %history for input ranges,
201 This function uses the same syntax as %history for input ranges,
202 then saves the lines to the filename you specify.
202 then saves the lines to the filename you specify.
203
203
204 It adds a '.py' extension to the file if you don't do so yourself, and
204 It adds a '.py' extension to the file if you don't do so yourself, and
205 it asks for confirmation before overwriting existing files.
205 it asks for confirmation before overwriting existing files.
206
206
207 If `-r` option is used, the default extension is `.ipy`.
207 If `-r` option is used, the default extension is `.ipy`.
208 """
208 """
209
209
210 opts,args = self.parse_options(parameter_s,'fra',mode='list')
210 opts,args = self.parse_options(parameter_s,'fra',mode='list')
211 if not args:
211 if not args:
212 raise UsageError('Missing filename.')
212 raise UsageError('Missing filename.')
213 raw = 'r' in opts
213 raw = 'r' in opts
214 force = 'f' in opts
214 force = 'f' in opts
215 append = 'a' in opts
215 append = 'a' in opts
216 mode = 'a' if append else 'w'
216 mode = 'a' if append else 'w'
217 ext = u'.ipy' if raw else u'.py'
217 ext = u'.ipy' if raw else u'.py'
218 fname, codefrom = args[0], " ".join(args[1:])
218 fname, codefrom = args[0], " ".join(args[1:])
219 if not fname.endswith((u'.py',u'.ipy')):
219 if not fname.endswith((u'.py',u'.ipy')):
220 fname += ext
220 fname += ext
221 file_exists = os.path.isfile(fname)
221 file_exists = os.path.isfile(fname)
222 if file_exists and not force and not append:
222 if file_exists and not force and not append:
223 try:
223 try:
224 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
224 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
225 except StdinNotImplementedError:
225 except StdinNotImplementedError:
226 print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s))
226 print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s))
227 return
227 return
228 if not overwrite :
228 if not overwrite :
229 print('Operation cancelled.')
229 print('Operation cancelled.')
230 return
230 return
231 try:
231 try:
232 cmds = self.shell.find_user_code(codefrom,raw)
232 cmds = self.shell.find_user_code(codefrom,raw)
233 except (TypeError, ValueError) as e:
233 except (TypeError, ValueError) as e:
234 print(e.args[0])
234 print(e.args[0])
235 return
235 return
236 out = py3compat.cast_unicode(cmds)
236 out = py3compat.cast_unicode(cmds)
237 with io.open(fname, mode, encoding="utf-8") as f:
237 with io.open(fname, mode, encoding="utf-8") as f:
238 if not file_exists or not append:
238 if not file_exists or not append:
239 f.write(u"# coding: utf-8\n")
239 f.write(u"# coding: utf-8\n")
240 f.write(out)
240 f.write(out)
241 # make sure we end on a newline
241 # make sure we end on a newline
242 if not out.endswith(u'\n'):
242 if not out.endswith(u'\n'):
243 f.write(u'\n')
243 f.write(u'\n')
244 print('The following commands were written to file `%s`:' % fname)
244 print('The following commands were written to file `%s`:' % fname)
245 print(cmds)
245 print(cmds)
246
246
247 @line_magic
247 @line_magic
248 def pastebin(self, parameter_s=''):
248 def pastebin(self, parameter_s=''):
249 """Upload code to dpaste's paste bin, returning the URL.
249 """Upload code to dpaste's paste bin, returning the URL.
250
250
251 Usage:\\
251 Usage:\\
252 %pastebin [-d "Custom description"] 1-7
252 %pastebin [-d "Custom description"] 1-7
253
253
254 The argument can be an input history range, a filename, or the name of a
254 The argument can be an input history range, a filename, or the name of a
255 string or macro.
255 string or macro.
256
256
257 Options:
257 Options:
258
258
259 -d: Pass a custom description for the gist. The default will say
259 -d: Pass a custom description for the gist. The default will say
260 "Pasted from IPython".
260 "Pasted from IPython".
261 """
261 """
262 opts, args = self.parse_options(parameter_s, 'd:')
262 opts, args = self.parse_options(parameter_s, 'd:')
263
263
264 try:
264 try:
265 code = self.shell.find_user_code(args)
265 code = self.shell.find_user_code(args)
266 except (ValueError, TypeError) as e:
266 except (ValueError, TypeError) as e:
267 print(e.args[0])
267 print(e.args[0])
268 return
268 return
269
269
270 post_data = urlencode({
270 post_data = urlencode({
271 "title": opts.get('d', "Pasted from IPython"),
271 "title": opts.get('d', "Pasted from IPython"),
272 "syntax": "python3",
272 "syntax": "python3",
273 "content": code
273 "content": code
274 }).encode('utf-8')
274 }).encode('utf-8')
275
275
276 response = urlopen("http://dpaste.com/api/v2/", post_data)
276 response = urlopen("http://dpaste.com/api/v2/", post_data)
277 return response.headers.get('Location')
277 return response.headers.get('Location')
278
278
279 @line_magic
279 @line_magic
280 def loadpy(self, arg_s):
280 def loadpy(self, arg_s):
281 """Alias of `%load`
281 """Alias of `%load`
282
282
283 `%loadpy` has gained some flexibility and dropped the requirement of a `.py`
283 `%loadpy` has gained some flexibility and dropped the requirement of a `.py`
284 extension. So it has been renamed simply into %load. You can look at
284 extension. So it has been renamed simply into %load. You can look at
285 `%load`'s docstring for more info.
285 `%load`'s docstring for more info.
286 """
286 """
287 self.load(arg_s)
287 self.load(arg_s)
288
288
289 @line_magic
289 @line_magic
290 def load(self, arg_s):
290 def load(self, arg_s):
291 """Load code into the current frontend.
291 """Load code into the current frontend.
292
292
293 Usage:\\
293 Usage:\\
294 %load [options] source
294 %load [options] source
295
295
296 where source can be a filename, URL, input history range, macro, or
296 where source can be a filename, URL, input history range, macro, or
297 element in the user namespace
297 element in the user namespace
298
298
299 Options:
299 Options:
300
300
301 -r <lines>: Specify lines or ranges of lines to load from the source.
301 -r <lines>: Specify lines or ranges of lines to load from the source.
302 Ranges could be specified as x-y (x..y) or in python-style x:y
302 Ranges could be specified as x-y (x..y) or in python-style x:y
303 (x..(y-1)). Both limits x and y can be left blank (meaning the
303 (x..(y-1)). Both limits x and y can be left blank (meaning the
304 beginning and end of the file, respectively).
304 beginning and end of the file, respectively).
305
305
306 -s <symbols>: Specify function or classes to load from python source.
306 -s <symbols>: Specify function or classes to load from python source.
307
307
308 -y : Don't ask confirmation for loading source above 200 000 characters.
308 -y : Don't ask confirmation for loading source above 200 000 characters.
309
309
310 -n : Include the user's namespace when searching for source code.
310 -n : Include the user's namespace when searching for source code.
311
311
312 This magic command can either take a local filename, a URL, an history
312 This magic command can either take a local filename, a URL, an history
313 range (see %history) or a macro as argument, it will prompt for
313 range (see %history) or a macro as argument, it will prompt for
314 confirmation before loading source with more than 200 000 characters, unless
314 confirmation before loading source with more than 200 000 characters, unless
315 -y flag is passed or if the frontend does not support raw_input::
315 -y flag is passed or if the frontend does not support raw_input::
316
316
317 %load myscript.py
317 %load myscript.py
318 %load 7-27
318 %load 7-27
319 %load myMacro
319 %load myMacro
320 %load http://www.example.com/myscript.py
320 %load http://www.example.com/myscript.py
321 %load -r 5-10 myscript.py
321 %load -r 5-10 myscript.py
322 %load -r 10-20,30,40: foo.py
322 %load -r 10-20,30,40: foo.py
323 %load -s MyClass,wonder_function myscript.py
323 %load -s MyClass,wonder_function myscript.py
324 %load -n MyClass
324 %load -n MyClass
325 %load -n my_module.wonder_function
325 %load -n my_module.wonder_function
326 """
326 """
327 opts,args = self.parse_options(arg_s,'yns:r:')
327 opts,args = self.parse_options(arg_s,'yns:r:')
328
328
329 if not args:
329 if not args:
330 raise UsageError('Missing filename, URL, input history range, '
330 raise UsageError('Missing filename, URL, input history range, '
331 'macro, or element in the user namespace.')
331 'macro, or element in the user namespace.')
332
332
333 search_ns = 'n' in opts
333 search_ns = 'n' in opts
334
334
335 contents = self.shell.find_user_code(args, search_ns=search_ns)
335 contents = self.shell.find_user_code(args, search_ns=search_ns)
336
336
337 if 's' in opts:
337 if 's' in opts:
338 try:
338 try:
339 blocks, not_found = extract_symbols(contents, opts['s'])
339 blocks, not_found = extract_symbols(contents, opts['s'])
340 except SyntaxError:
340 except SyntaxError:
341 # non python code
341 # non python code
342 error("Unable to parse the input as valid Python code")
342 error("Unable to parse the input as valid Python code")
343 return
343 return
344
344
345 if len(not_found) == 1:
345 if len(not_found) == 1:
346 warn('The symbol `%s` was not found' % not_found[0])
346 warn('The symbol `%s` was not found' % not_found[0])
347 elif len(not_found) > 1:
347 elif len(not_found) > 1:
348 warn('The symbols %s were not found' % get_text_list(not_found,
348 warn('The symbols %s were not found' % get_text_list(not_found,
349 wrap_item_with='`')
349 wrap_item_with='`')
350 )
350 )
351
351
352 contents = '\n'.join(blocks)
352 contents = '\n'.join(blocks)
353
353
354 if 'r' in opts:
354 if 'r' in opts:
355 ranges = opts['r'].replace(',', ' ')
355 ranges = opts['r'].replace(',', ' ')
356 lines = contents.split('\n')
356 lines = contents.split('\n')
357 slices = extract_code_ranges(ranges)
357 slices = extract_code_ranges(ranges)
358 contents = [lines[slice(*slc)] for slc in slices]
358 contents = [lines[slice(*slc)] for slc in slices]
359 contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents)))
359 contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents)))
360
360
361 l = len(contents)
361 l = len(contents)
362
362
363 # 200 000 is ~ 2500 full 80 character lines
363 # 200 000 is ~ 2500 full 80 character lines
364 # so in average, more than 5000 lines
364 # so in average, more than 5000 lines
365 if l > 200000 and 'y' not in opts:
365 if l > 200000 and 'y' not in opts:
366 try:
366 try:
367 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
367 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
368 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
368 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
369 except StdinNotImplementedError:
369 except StdinNotImplementedError:
370 #assume yes if raw input not implemented
370 #assume yes if raw input not implemented
371 ans = True
371 ans = True
372
372
373 if ans is False :
373 if ans is False :
374 print('Operation cancelled.')
374 print('Operation cancelled.')
375 return
375 return
376
376
377 contents = "# %load {}\n".format(arg_s) + contents
377 contents = "# %load {}\n".format(arg_s) + contents
378
378
379 self.shell.set_next_input(contents, replace=True)
379 self.shell.set_next_input(contents, replace=True)
380
380
381 @staticmethod
381 @staticmethod
382 def _find_edit_target(shell, args, opts, last_call):
382 def _find_edit_target(shell, args, opts, last_call):
383 """Utility method used by magic_edit to find what to edit."""
383 """Utility method used by magic_edit to find what to edit."""
384
384
385 def make_filename(arg):
385 def make_filename(arg):
386 "Make a filename from the given args"
386 "Make a filename from the given args"
387 try:
387 try:
388 filename = get_py_filename(arg)
388 filename = get_py_filename(arg)
389 except IOError:
389 except IOError:
390 # If it ends with .py but doesn't already exist, assume we want
390 # If it ends with .py but doesn't already exist, assume we want
391 # a new file.
391 # a new file.
392 if arg.endswith('.py'):
392 if arg.endswith('.py'):
393 filename = arg
393 filename = arg
394 else:
394 else:
395 filename = None
395 filename = None
396 return filename
396 return filename
397
397
398 # Set a few locals from the options for convenience:
398 # Set a few locals from the options for convenience:
399 opts_prev = 'p' in opts
399 opts_prev = 'p' in opts
400 opts_raw = 'r' in opts
400 opts_raw = 'r' in opts
401
401
402 # custom exceptions
402 # custom exceptions
403 class DataIsObject(Exception): pass
403 class DataIsObject(Exception): pass
404
404
405 # Default line number value
405 # Default line number value
406 lineno = opts.get('n',None)
406 lineno = opts.get('n',None)
407
407
408 if opts_prev:
408 if opts_prev:
409 args = '_%s' % last_call[0]
409 args = '_%s' % last_call[0]
410 if args not in shell.user_ns:
410 if args not in shell.user_ns:
411 args = last_call[1]
411 args = last_call[1]
412
412
413 # by default this is done with temp files, except when the given
413 # by default this is done with temp files, except when the given
414 # arg is a filename
414 # arg is a filename
415 use_temp = True
415 use_temp = True
416
416
417 data = ''
417 data = ''
418
418
419 # First, see if the arguments should be a filename.
419 # First, see if the arguments should be a filename.
420 filename = make_filename(args)
420 filename = make_filename(args)
421 if filename:
421 if filename:
422 use_temp = False
422 use_temp = False
423 elif args:
423 elif args:
424 # Mode where user specifies ranges of lines, like in %macro.
424 # Mode where user specifies ranges of lines, like in %macro.
425 data = shell.extract_input_lines(args, opts_raw)
425 data = shell.extract_input_lines(args, opts_raw)
426 if not data:
426 if not data:
427 try:
427 try:
428 # Load the parameter given as a variable. If not a string,
428 # Load the parameter given as a variable. If not a string,
429 # process it as an object instead (below)
429 # process it as an object instead (below)
430
430
431 #print '*** args',args,'type',type(args) # dbg
431 #print '*** args',args,'type',type(args) # dbg
432 data = eval(args, shell.user_ns)
432 data = eval(args, shell.user_ns)
433 if not isinstance(data, str):
433 if not isinstance(data, str):
434 raise DataIsObject
434 raise DataIsObject
435
435
436 except (NameError,SyntaxError):
436 except (NameError,SyntaxError):
437 # given argument is not a variable, try as a filename
437 # given argument is not a variable, try as a filename
438 filename = make_filename(args)
438 filename = make_filename(args)
439 if filename is None:
439 if filename is None:
440 warn("Argument given (%s) can't be found as a variable "
440 warn("Argument given (%s) can't be found as a variable "
441 "or as a filename." % args)
441 "or as a filename." % args)
442 return (None, None, None)
442 return (None, None, None)
443 use_temp = False
443 use_temp = False
444
444
445 except DataIsObject:
445 except DataIsObject:
446 # macros have a special edit function
446 # macros have a special edit function
447 if isinstance(data, Macro):
447 if isinstance(data, Macro):
448 raise MacroToEdit(data)
448 raise MacroToEdit(data)
449
449
450 # For objects, try to edit the file where they are defined
450 # For objects, try to edit the file where they are defined
451 filename = find_file(data)
451 filename = find_file(data)
452 if filename:
452 if filename:
453 if 'fakemodule' in filename.lower() and \
453 if 'fakemodule' in filename.lower() and \
454 inspect.isclass(data):
454 inspect.isclass(data):
455 # class created by %edit? Try to find source
455 # class created by %edit? Try to find source
456 # by looking for method definitions instead, the
456 # by looking for method definitions instead, the
457 # __module__ in those classes is FakeModule.
457 # __module__ in those classes is FakeModule.
458 attrs = [getattr(data, aname) for aname in dir(data)]
458 attrs = [getattr(data, aname) for aname in dir(data)]
459 for attr in attrs:
459 for attr in attrs:
460 if not inspect.ismethod(attr):
460 if not inspect.ismethod(attr):
461 continue
461 continue
462 filename = find_file(attr)
462 filename = find_file(attr)
463 if filename and \
463 if filename and \
464 'fakemodule' not in filename.lower():
464 'fakemodule' not in filename.lower():
465 # change the attribute to be the edit
465 # change the attribute to be the edit
466 # target instead
466 # target instead
467 data = attr
467 data = attr
468 break
468 break
469
469
470 m = ipython_input_pat.match(os.path.basename(filename))
470 m = ipython_input_pat.match(os.path.basename(filename))
471 if m:
471 if m:
472 raise InteractivelyDefined(int(m.groups()[0]))
472 raise InteractivelyDefined(int(m.groups()[0]))
473
473
474 datafile = 1
474 datafile = 1
475 if filename is None:
475 if filename is None:
476 filename = make_filename(args)
476 filename = make_filename(args)
477 datafile = 1
477 datafile = 1
478 if filename is not None:
478 if filename is not None:
479 # only warn about this if we get a real name
479 # only warn about this if we get a real name
480 warn('Could not find file where `%s` is defined.\n'
480 warn('Could not find file where `%s` is defined.\n'
481 'Opening a file named `%s`' % (args, filename))
481 'Opening a file named `%s`' % (args, filename))
482 # Now, make sure we can actually read the source (if it was
482 # Now, make sure we can actually read the source (if it was
483 # in a temp file it's gone by now).
483 # in a temp file it's gone by now).
484 if datafile:
484 if datafile:
485 if lineno is None:
485 if lineno is None:
486 lineno = find_source_lines(data)
486 lineno = find_source_lines(data)
487 if lineno is None:
487 if lineno is None:
488 filename = make_filename(args)
488 filename = make_filename(args)
489 if filename is None:
489 if filename is None:
490 warn('The file where `%s` was defined '
490 warn('The file where `%s` was defined '
491 'cannot be read or found.' % data)
491 'cannot be read or found.' % data)
492 return (None, None, None)
492 return (None, None, None)
493 use_temp = False
493 use_temp = False
494
494
495 if use_temp:
495 if use_temp:
496 filename = shell.mktempfile(data)
496 filename = shell.mktempfile(data)
497 print('IPython will make a temporary file named:',filename)
497 print('IPython will make a temporary file named:',filename)
498
498
499 # use last_call to remember the state of the previous call, but don't
499 # use last_call to remember the state of the previous call, but don't
500 # let it be clobbered by successive '-p' calls.
500 # let it be clobbered by successive '-p' calls.
501 try:
501 try:
502 last_call[0] = shell.displayhook.prompt_count
502 last_call[0] = shell.displayhook.prompt_count
503 if not opts_prev:
503 if not opts_prev:
504 last_call[1] = args
504 last_call[1] = args
505 except:
505 except:
506 pass
506 pass
507
507
508
508
509 return filename, lineno, use_temp
509 return filename, lineno, use_temp
510
510
511 def _edit_macro(self,mname,macro):
511 def _edit_macro(self,mname,macro):
512 """open an editor with the macro data in a file"""
512 """open an editor with the macro data in a file"""
513 filename = self.shell.mktempfile(macro.value)
513 filename = self.shell.mktempfile(macro.value)
514 self.shell.hooks.editor(filename)
514 self.shell.hooks.editor(filename)
515
515
516 # and make a new macro object, to replace the old one
516 # and make a new macro object, to replace the old one
517 with open(filename) as mfile:
517 with open(filename) as mfile:
518 mvalue = mfile.read()
518 mvalue = mfile.read()
519 self.shell.user_ns[mname] = Macro(mvalue)
519 self.shell.user_ns[mname] = Macro(mvalue)
520
520
521 @skip_doctest
521 @skip_doctest
522 @line_magic
522 @line_magic
523 def edit(self, parameter_s='',last_call=['','']):
523 def edit(self, parameter_s='',last_call=['','']):
524 """Bring up an editor and execute the resulting code.
524 """Bring up an editor and execute the resulting code.
525
525
526 Usage:
526 Usage:
527 %edit [options] [args]
527 %edit [options] [args]
528
528
529 %edit runs IPython's editor hook. The default version of this hook is
529 %edit runs IPython's editor hook. The default version of this hook is
530 set to call the editor specified by your $EDITOR environment variable.
530 set to call the editor specified by your $EDITOR environment variable.
531 If this isn't found, it will default to vi under Linux/Unix and to
531 If this isn't found, it will default to vi under Linux/Unix and to
532 notepad under Windows. See the end of this docstring for how to change
532 notepad under Windows. See the end of this docstring for how to change
533 the editor hook.
533 the editor hook.
534
534
535 You can also set the value of this editor via the
535 You can also set the value of this editor via the
536 ``TerminalInteractiveShell.editor`` option in your configuration file.
536 ``TerminalInteractiveShell.editor`` option in your configuration file.
537 This is useful if you wish to use a different editor from your typical
537 This is useful if you wish to use a different editor from your typical
538 default with IPython (and for Windows users who typically don't set
538 default with IPython (and for Windows users who typically don't set
539 environment variables).
539 environment variables).
540
540
541 This command allows you to conveniently edit multi-line code right in
541 This command allows you to conveniently edit multi-line code right in
542 your IPython session.
542 your IPython session.
543
543
544 If called without arguments, %edit opens up an empty editor with a
544 If called without arguments, %edit opens up an empty editor with a
545 temporary file and will execute the contents of this file when you
545 temporary file and will execute the contents of this file when you
546 close it (don't forget to save it!).
546 close it (don't forget to save it!).
547
547
548
548
549 Options:
549 Options:
550
550
551 -n <number>: open the editor at a specified line number. By default,
551 -n <number>: open the editor at a specified line number. By default,
552 the IPython editor hook uses the unix syntax 'editor +N filename', but
552 the IPython editor hook uses the unix syntax 'editor +N filename', but
553 you can configure this by providing your own modified hook if your
553 you can configure this by providing your own modified hook if your
554 favorite editor supports line-number specifications with a different
554 favorite editor supports line-number specifications with a different
555 syntax.
555 syntax.
556
556
557 -p: this will call the editor with the same data as the previous time
557 -p: this will call the editor with the same data as the previous time
558 it was used, regardless of how long ago (in your current session) it
558 it was used, regardless of how long ago (in your current session) it
559 was.
559 was.
560
560
561 -r: use 'raw' input. This option only applies to input taken from the
561 -r: use 'raw' input. This option only applies to input taken from the
562 user's history. By default, the 'processed' history is used, so that
562 user's history. By default, the 'processed' history is used, so that
563 magics are loaded in their transformed version to valid Python. If
563 magics are loaded in their transformed version to valid Python. If
564 this option is given, the raw input as typed as the command line is
564 this option is given, the raw input as typed as the command line is
565 used instead. When you exit the editor, it will be executed by
565 used instead. When you exit the editor, it will be executed by
566 IPython's own processor.
566 IPython's own processor.
567
567
568 -x: do not execute the edited code immediately upon exit. This is
568 -x: do not execute the edited code immediately upon exit. This is
569 mainly useful if you are editing programs which need to be called with
569 mainly useful if you are editing programs which need to be called with
570 command line arguments, which you can then do using %run.
570 command line arguments, which you can then do using %run.
571
571
572
572
573 Arguments:
573 Arguments:
574
574
575 If arguments are given, the following possibilities exist:
575 If arguments are given, the following possibilities exist:
576
576
577 - If the argument is a filename, IPython will load that into the
577 - If the argument is a filename, IPython will load that into the
578 editor. It will execute its contents with execfile() when you exit,
578 editor. It will execute its contents with execfile() when you exit,
579 loading any code in the file into your interactive namespace.
579 loading any code in the file into your interactive namespace.
580
580
581 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
581 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
582 The syntax is the same as in the %history magic.
582 The syntax is the same as in the %history magic.
583
583
584 - If the argument is a string variable, its contents are loaded
584 - If the argument is a string variable, its contents are loaded
585 into the editor. You can thus edit any string which contains
585 into the editor. You can thus edit any string which contains
586 python code (including the result of previous edits).
586 python code (including the result of previous edits).
587
587
588 - If the argument is the name of an object (other than a string),
588 - If the argument is the name of an object (other than a string),
589 IPython will try to locate the file where it was defined and open the
589 IPython will try to locate the file where it was defined and open the
590 editor at the point where it is defined. You can use `%edit function`
590 editor at the point where it is defined. You can use `%edit function`
591 to load an editor exactly at the point where 'function' is defined,
591 to load an editor exactly at the point where 'function' is defined,
592 edit it and have the file be executed automatically.
592 edit it and have the file be executed automatically.
593
593
594 - If the object is a macro (see %macro for details), this opens up your
594 - If the object is a macro (see %macro for details), this opens up your
595 specified editor with a temporary file containing the macro's data.
595 specified editor with a temporary file containing the macro's data.
596 Upon exit, the macro is reloaded with the contents of the file.
596 Upon exit, the macro is reloaded with the contents of the file.
597
597
598 Note: opening at an exact line is only supported under Unix, and some
598 Note: opening at an exact line is only supported under Unix, and some
599 editors (like kedit and gedit up to Gnome 2.8) do not understand the
599 editors (like kedit and gedit up to Gnome 2.8) do not understand the
600 '+NUMBER' parameter necessary for this feature. Good editors like
600 '+NUMBER' parameter necessary for this feature. Good editors like
601 (X)Emacs, vi, jed, pico and joe all do.
601 (X)Emacs, vi, jed, pico and joe all do.
602
602
603 After executing your code, %edit will return as output the code you
603 After executing your code, %edit will return as output the code you
604 typed in the editor (except when it was an existing file). This way
604 typed in the editor (except when it was an existing file). This way
605 you can reload the code in further invocations of %edit as a variable,
605 you can reload the code in further invocations of %edit as a variable,
606 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
606 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
607 the output.
607 the output.
608
608
609 Note that %edit is also available through the alias %ed.
609 Note that %edit is also available through the alias %ed.
610
610
611 This is an example of creating a simple function inside the editor and
611 This is an example of creating a simple function inside the editor and
612 then modifying it. First, start up the editor::
612 then modifying it. First, start up the editor::
613
613
614 In [1]: edit
614 In [1]: edit
615 Editing... done. Executing edited code...
615 Editing... done. Executing edited code...
616 Out[1]: 'def foo():\\n print "foo() was defined in an editing
616 Out[1]: 'def foo():\\n print "foo() was defined in an editing
617 session"\\n'
617 session"\\n'
618
618
619 We can then call the function foo()::
619 We can then call the function foo()::
620
620
621 In [2]: foo()
621 In [2]: foo()
622 foo() was defined in an editing session
622 foo() was defined in an editing session
623
623
624 Now we edit foo. IPython automatically loads the editor with the
624 Now we edit foo. IPython automatically loads the editor with the
625 (temporary) file where foo() was previously defined::
625 (temporary) file where foo() was previously defined::
626
626
627 In [3]: edit foo
627 In [3]: edit foo
628 Editing... done. Executing edited code...
628 Editing... done. Executing edited code...
629
629
630 And if we call foo() again we get the modified version::
630 And if we call foo() again we get the modified version::
631
631
632 In [4]: foo()
632 In [4]: foo()
633 foo() has now been changed!
633 foo() has now been changed!
634
634
635 Here is an example of how to edit a code snippet successive
635 Here is an example of how to edit a code snippet successive
636 times. First we call the editor::
636 times. First we call the editor::
637
637
638 In [5]: edit
638 In [5]: edit
639 Editing... done. Executing edited code...
639 Editing... done. Executing edited code...
640 hello
640 hello
641 Out[5]: "print 'hello'\\n"
641 Out[5]: "print 'hello'\\n"
642
642
643 Now we call it again with the previous output (stored in _)::
643 Now we call it again with the previous output (stored in _)::
644
644
645 In [6]: edit _
645 In [6]: edit _
646 Editing... done. Executing edited code...
646 Editing... done. Executing edited code...
647 hello world
647 hello world
648 Out[6]: "print 'hello world'\\n"
648 Out[6]: "print 'hello world'\\n"
649
649
650 Now we call it with the output #8 (stored in _8, also as Out[8])::
650 Now we call it with the output #8 (stored in _8, also as Out[8])::
651
651
652 In [7]: edit _8
652 In [7]: edit _8
653 Editing... done. Executing edited code...
653 Editing... done. Executing edited code...
654 hello again
654 hello again
655 Out[7]: "print 'hello again'\\n"
655 Out[7]: "print 'hello again'\\n"
656
656
657
657
658 Changing the default editor hook:
658 Changing the default editor hook:
659
659
660 If you wish to write your own editor hook, you can put it in a
660 If you wish to write your own editor hook, you can put it in a
661 configuration file which you load at startup time. The default hook
661 configuration file which you load at startup time. The default hook
662 is defined in the IPython.core.hooks module, and you can use that as a
662 is defined in the IPython.core.hooks module, and you can use that as a
663 starting example for further modifications. That file also has
663 starting example for further modifications. That file also has
664 general instructions on how to set a new hook for use once you've
664 general instructions on how to set a new hook for use once you've
665 defined it."""
665 defined it."""
666 opts,args = self.parse_options(parameter_s,'prxn:')
666 opts,args = self.parse_options(parameter_s,'prxn:')
667
667
668 try:
668 try:
669 filename, lineno, is_temp = self._find_edit_target(self.shell,
669 filename, lineno, is_temp = self._find_edit_target(self.shell,
670 args, opts, last_call)
670 args, opts, last_call)
671 except MacroToEdit as e:
671 except MacroToEdit as e:
672 self._edit_macro(args, e.args[0])
672 self._edit_macro(args, e.args[0])
673 return
673 return
674 except InteractivelyDefined as e:
674 except InteractivelyDefined as e:
675 print("Editing In[%i]" % e.index)
675 print("Editing In[%i]" % e.index)
676 args = str(e.index)
676 args = str(e.index)
677 filename, lineno, is_temp = self._find_edit_target(self.shell,
677 filename, lineno, is_temp = self._find_edit_target(self.shell,
678 args, opts, last_call)
678 args, opts, last_call)
679 if filename is None:
679 if filename is None:
680 # nothing was found, warnings have already been issued,
680 # nothing was found, warnings have already been issued,
681 # just give up.
681 # just give up.
682 return
682 return
683
683
684 if is_temp:
684 if is_temp:
685 self._knowntemps.add(filename)
685 self._knowntemps.add(filename)
686 elif (filename in self._knowntemps):
686 elif (filename in self._knowntemps):
687 is_temp = True
687 is_temp = True
688
688
689
689
690 # do actual editing here
690 # do actual editing here
691 print('Editing...', end=' ')
691 print('Editing...', end=' ')
692 sys.stdout.flush()
692 sys.stdout.flush()
693 try:
693 try:
694 # Quote filenames that may have spaces in them
694 # Quote filenames that may have spaces in them
695 if ' ' in filename:
695 if ' ' in filename:
696 filename = "'%s'" % filename
696 filename = "'%s'" % filename
697 self.shell.hooks.editor(filename,lineno)
697 self.shell.hooks.editor(filename,lineno)
698 except TryNext:
698 except TryNext:
699 warn('Could not open editor')
699 warn('Could not open editor')
700 return
700 return
701
701
702 # XXX TODO: should this be generalized for all string vars?
702 # XXX TODO: should this be generalized for all string vars?
703 # For now, this is special-cased to blocks created by cpaste
703 # For now, this is special-cased to blocks created by cpaste
704 if args.strip() == 'pasted_block':
704 if args.strip() == 'pasted_block':
705 with open(filename, 'r') as f:
705 with open(filename, 'r') as f:
706 self.shell.user_ns['pasted_block'] = f.read()
706 self.shell.user_ns['pasted_block'] = f.read()
707
707
708 if 'x' in opts: # -x prevents actual execution
708 if 'x' in opts: # -x prevents actual execution
709 print()
709 print()
710 else:
710 else:
711 print('done. Executing edited code...')
711 print('done. Executing edited code...')
712 with preserve_keys(self.shell.user_ns, '__file__'):
712 with preserve_keys(self.shell.user_ns, '__file__'):
713 if not is_temp:
713 if not is_temp:
714 self.shell.user_ns['__file__'] = filename
714 self.shell.user_ns['__file__'] = filename
715 if 'r' in opts: # Untranslated IPython code
715 if 'r' in opts: # Untranslated IPython code
716 with open(filename, 'r') as f:
716 with open(filename, 'r') as f:
717 source = f.read()
717 source = f.read()
718 self.shell.run_cell(source, store_history=False)
718 self.shell.run_cell(source, store_history=False)
719 else:
719 else:
720 self.shell.safe_execfile(filename, self.shell.user_ns,
720 self.shell.safe_execfile(filename, self.shell.user_ns,
721 self.shell.user_ns)
721 self.shell.user_ns)
722
722
723 if is_temp:
723 if is_temp:
724 try:
724 try:
725 return open(filename).read()
725 with open(filename) as f:
726 return f.read()
726 except IOError as msg:
727 except IOError as msg:
727 if msg.filename == filename:
728 if msg.filename == filename:
728 warn('File not found. Did you forget to save?')
729 warn('File not found. Did you forget to save?')
729 return
730 return
730 else:
731 else:
731 self.shell.showtraceback()
732 self.shell.showtraceback()
@@ -1,1479 +1,1478 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Implementation of execution-related magic functions."""
2 """Implementation of execution-related magic functions."""
3
3
4 # Copyright (c) IPython Development Team.
4 # Copyright (c) IPython Development Team.
5 # Distributed under the terms of the Modified BSD License.
5 # Distributed under the terms of the Modified BSD License.
6
6
7
7
8 import ast
8 import ast
9 import bdb
9 import bdb
10 import builtins as builtin_mod
10 import builtins as builtin_mod
11 import gc
11 import gc
12 import itertools
12 import itertools
13 import os
13 import os
14 import shlex
14 import shlex
15 import sys
15 import sys
16 import time
16 import time
17 import timeit
17 import timeit
18 import math
18 import math
19 import re
19 import re
20 from pdb import Restart
20 from pdb import Restart
21
21
22 # cProfile was added in Python2.5
22 # cProfile was added in Python2.5
23 try:
23 try:
24 import cProfile as profile
24 import cProfile as profile
25 import pstats
25 import pstats
26 except ImportError:
26 except ImportError:
27 # profile isn't bundled by default in Debian for license reasons
27 # profile isn't bundled by default in Debian for license reasons
28 try:
28 try:
29 import profile, pstats
29 import profile, pstats
30 except ImportError:
30 except ImportError:
31 profile = pstats = None
31 profile = pstats = None
32
32
33 from IPython.core import oinspect
33 from IPython.core import oinspect
34 from IPython.core import magic_arguments
34 from IPython.core import magic_arguments
35 from IPython.core import page
35 from IPython.core import page
36 from IPython.core.error import UsageError
36 from IPython.core.error import UsageError
37 from IPython.core.macro import Macro
37 from IPython.core.macro import Macro
38 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
38 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
39 line_cell_magic, on_off, needs_local_scope,
39 line_cell_magic, on_off, needs_local_scope,
40 no_var_expand)
40 no_var_expand)
41 from IPython.testing.skipdoctest import skip_doctest
41 from IPython.testing.skipdoctest import skip_doctest
42 from IPython.utils.contexts import preserve_keys
42 from IPython.utils.contexts import preserve_keys
43 from IPython.utils.capture import capture_output
43 from IPython.utils.capture import capture_output
44 from IPython.utils.ipstruct import Struct
44 from IPython.utils.ipstruct import Struct
45 from IPython.utils.module_paths import find_mod
45 from IPython.utils.module_paths import find_mod
46 from IPython.utils.path import get_py_filename, shellglob
46 from IPython.utils.path import get_py_filename, shellglob
47 from IPython.utils.timing import clock, clock2
47 from IPython.utils.timing import clock, clock2
48 from warnings import warn
48 from warnings import warn
49 from logging import error
49 from logging import error
50 from io import StringIO
50 from io import StringIO
51
51
52
52
53 #-----------------------------------------------------------------------------
53 #-----------------------------------------------------------------------------
54 # Magic implementation classes
54 # Magic implementation classes
55 #-----------------------------------------------------------------------------
55 #-----------------------------------------------------------------------------
56
56
57
57
58 class TimeitResult(object):
58 class TimeitResult(object):
59 """
59 """
60 Object returned by the timeit magic with info about the run.
60 Object returned by the timeit magic with info about the run.
61
61
62 Contains the following attributes :
62 Contains the following attributes :
63
63
64 loops: (int) number of loops done per measurement
64 loops: (int) number of loops done per measurement
65 repeat: (int) number of times the measurement has been repeated
65 repeat: (int) number of times the measurement has been repeated
66 best: (float) best execution time / number
66 best: (float) best execution time / number
67 all_runs: (list of float) execution time of each run (in s)
67 all_runs: (list of float) execution time of each run (in s)
68 compile_time: (float) time of statement compilation (s)
68 compile_time: (float) time of statement compilation (s)
69
69
70 """
70 """
71 def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
71 def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
72 self.loops = loops
72 self.loops = loops
73 self.repeat = repeat
73 self.repeat = repeat
74 self.best = best
74 self.best = best
75 self.worst = worst
75 self.worst = worst
76 self.all_runs = all_runs
76 self.all_runs = all_runs
77 self.compile_time = compile_time
77 self.compile_time = compile_time
78 self._precision = precision
78 self._precision = precision
79 self.timings = [ dt / self.loops for dt in all_runs]
79 self.timings = [ dt / self.loops for dt in all_runs]
80
80
81 @property
81 @property
82 def average(self):
82 def average(self):
83 return math.fsum(self.timings) / len(self.timings)
83 return math.fsum(self.timings) / len(self.timings)
84
84
85 @property
85 @property
86 def stdev(self):
86 def stdev(self):
87 mean = self.average
87 mean = self.average
88 return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
88 return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
89
89
90 def __str__(self):
90 def __str__(self):
91 pm = '+-'
91 pm = '+-'
92 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
92 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
93 try:
93 try:
94 u'\xb1'.encode(sys.stdout.encoding)
94 u'\xb1'.encode(sys.stdout.encoding)
95 pm = u'\xb1'
95 pm = u'\xb1'
96 except:
96 except:
97 pass
97 pass
98 return (
98 return (
99 u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
99 u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
100 .format(
100 .format(
101 pm = pm,
101 pm = pm,
102 runs = self.repeat,
102 runs = self.repeat,
103 loops = self.loops,
103 loops = self.loops,
104 loop_plural = "" if self.loops == 1 else "s",
104 loop_plural = "" if self.loops == 1 else "s",
105 run_plural = "" if self.repeat == 1 else "s",
105 run_plural = "" if self.repeat == 1 else "s",
106 mean = _format_time(self.average, self._precision),
106 mean = _format_time(self.average, self._precision),
107 std = _format_time(self.stdev, self._precision))
107 std = _format_time(self.stdev, self._precision))
108 )
108 )
109
109
110 def _repr_pretty_(self, p , cycle):
110 def _repr_pretty_(self, p , cycle):
111 unic = self.__str__()
111 unic = self.__str__()
112 p.text(u'<TimeitResult : '+unic+u'>')
112 p.text(u'<TimeitResult : '+unic+u'>')
113
113
114
114
115 class TimeitTemplateFiller(ast.NodeTransformer):
115 class TimeitTemplateFiller(ast.NodeTransformer):
116 """Fill in the AST template for timing execution.
116 """Fill in the AST template for timing execution.
117
117
118 This is quite closely tied to the template definition, which is in
118 This is quite closely tied to the template definition, which is in
119 :meth:`ExecutionMagics.timeit`.
119 :meth:`ExecutionMagics.timeit`.
120 """
120 """
121 def __init__(self, ast_setup, ast_stmt):
121 def __init__(self, ast_setup, ast_stmt):
122 self.ast_setup = ast_setup
122 self.ast_setup = ast_setup
123 self.ast_stmt = ast_stmt
123 self.ast_stmt = ast_stmt
124
124
125 def visit_FunctionDef(self, node):
125 def visit_FunctionDef(self, node):
126 "Fill in the setup statement"
126 "Fill in the setup statement"
127 self.generic_visit(node)
127 self.generic_visit(node)
128 if node.name == "inner":
128 if node.name == "inner":
129 node.body[:1] = self.ast_setup.body
129 node.body[:1] = self.ast_setup.body
130
130
131 return node
131 return node
132
132
133 def visit_For(self, node):
133 def visit_For(self, node):
134 "Fill in the statement to be timed"
134 "Fill in the statement to be timed"
135 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
135 if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
136 node.body = self.ast_stmt.body
136 node.body = self.ast_stmt.body
137 return node
137 return node
138
138
139
139
140 class Timer(timeit.Timer):
140 class Timer(timeit.Timer):
141 """Timer class that explicitly uses self.inner
141 """Timer class that explicitly uses self.inner
142
142
143 which is an undocumented implementation detail of CPython,
143 which is an undocumented implementation detail of CPython,
144 not shared by PyPy.
144 not shared by PyPy.
145 """
145 """
146 # Timer.timeit copied from CPython 3.4.2
146 # Timer.timeit copied from CPython 3.4.2
147 def timeit(self, number=timeit.default_number):
147 def timeit(self, number=timeit.default_number):
148 """Time 'number' executions of the main statement.
148 """Time 'number' executions of the main statement.
149
149
150 To be precise, this executes the setup statement once, and
150 To be precise, this executes the setup statement once, and
151 then returns the time it takes to execute the main statement
151 then returns the time it takes to execute the main statement
152 a number of times, as a float measured in seconds. The
152 a number of times, as a float measured in seconds. The
153 argument is the number of times through the loop, defaulting
153 argument is the number of times through the loop, defaulting
154 to one million. The main statement, the setup statement and
154 to one million. The main statement, the setup statement and
155 the timer function to be used are passed to the constructor.
155 the timer function to be used are passed to the constructor.
156 """
156 """
157 it = itertools.repeat(None, number)
157 it = itertools.repeat(None, number)
158 gcold = gc.isenabled()
158 gcold = gc.isenabled()
159 gc.disable()
159 gc.disable()
160 try:
160 try:
161 timing = self.inner(it, self.timer)
161 timing = self.inner(it, self.timer)
162 finally:
162 finally:
163 if gcold:
163 if gcold:
164 gc.enable()
164 gc.enable()
165 return timing
165 return timing
166
166
167
167
168 @magics_class
168 @magics_class
169 class ExecutionMagics(Magics):
169 class ExecutionMagics(Magics):
170 """Magics related to code execution, debugging, profiling, etc.
170 """Magics related to code execution, debugging, profiling, etc.
171
171
172 """
172 """
173
173
174 def __init__(self, shell):
174 def __init__(self, shell):
175 super(ExecutionMagics, self).__init__(shell)
175 super(ExecutionMagics, self).__init__(shell)
176 if profile is None:
176 if profile is None:
177 self.prun = self.profile_missing_notice
177 self.prun = self.profile_missing_notice
178 # Default execution function used to actually run user code.
178 # Default execution function used to actually run user code.
179 self.default_runner = None
179 self.default_runner = None
180
180
181 def profile_missing_notice(self, *args, **kwargs):
181 def profile_missing_notice(self, *args, **kwargs):
182 error("""\
182 error("""\
183 The profile module could not be found. It has been removed from the standard
183 The profile module could not be found. It has been removed from the standard
184 python packages because of its non-free license. To use profiling, install the
184 python packages because of its non-free license. To use profiling, install the
185 python-profiler package from non-free.""")
185 python-profiler package from non-free.""")
186
186
187 @skip_doctest
187 @skip_doctest
188 @no_var_expand
188 @no_var_expand
189 @line_cell_magic
189 @line_cell_magic
190 def prun(self, parameter_s='', cell=None):
190 def prun(self, parameter_s='', cell=None):
191
191
192 """Run a statement through the python code profiler.
192 """Run a statement through the python code profiler.
193
193
194 Usage, in line mode:
194 Usage, in line mode:
195 %prun [options] statement
195 %prun [options] statement
196
196
197 Usage, in cell mode:
197 Usage, in cell mode:
198 %%prun [options] [statement]
198 %%prun [options] [statement]
199 code...
199 code...
200 code...
200 code...
201
201
202 In cell mode, the additional code lines are appended to the (possibly
202 In cell mode, the additional code lines are appended to the (possibly
203 empty) statement in the first line. Cell mode allows you to easily
203 empty) statement in the first line. Cell mode allows you to easily
204 profile multiline blocks without having to put them in a separate
204 profile multiline blocks without having to put them in a separate
205 function.
205 function.
206
206
207 The given statement (which doesn't require quote marks) is run via the
207 The given statement (which doesn't require quote marks) is run via the
208 python profiler in a manner similar to the profile.run() function.
208 python profiler in a manner similar to the profile.run() function.
209 Namespaces are internally managed to work correctly; profile.run
209 Namespaces are internally managed to work correctly; profile.run
210 cannot be used in IPython because it makes certain assumptions about
210 cannot be used in IPython because it makes certain assumptions about
211 namespaces which do not hold under IPython.
211 namespaces which do not hold under IPython.
212
212
213 Options:
213 Options:
214
214
215 -l <limit>
215 -l <limit>
216 you can place restrictions on what or how much of the
216 you can place restrictions on what or how much of the
217 profile gets printed. The limit value can be:
217 profile gets printed. The limit value can be:
218
218
219 * A string: only information for function names containing this string
219 * A string: only information for function names containing this string
220 is printed.
220 is printed.
221
221
222 * An integer: only these many lines are printed.
222 * An integer: only these many lines are printed.
223
223
224 * A float (between 0 and 1): this fraction of the report is printed
224 * A float (between 0 and 1): this fraction of the report is printed
225 (for example, use a limit of 0.4 to see the topmost 40% only).
225 (for example, use a limit of 0.4 to see the topmost 40% only).
226
226
227 You can combine several limits with repeated use of the option. For
227 You can combine several limits with repeated use of the option. For
228 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
228 example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
229 information about class constructors.
229 information about class constructors.
230
230
231 -r
231 -r
232 return the pstats.Stats object generated by the profiling. This
232 return the pstats.Stats object generated by the profiling. This
233 object has all the information about the profile in it, and you can
233 object has all the information about the profile in it, and you can
234 later use it for further analysis or in other functions.
234 later use it for further analysis or in other functions.
235
235
236 -s <key>
236 -s <key>
237 sort profile by given key. You can provide more than one key
237 sort profile by given key. You can provide more than one key
238 by using the option several times: '-s key1 -s key2 -s key3...'. The
238 by using the option several times: '-s key1 -s key2 -s key3...'. The
239 default sorting key is 'time'.
239 default sorting key is 'time'.
240
240
241 The following is copied verbatim from the profile documentation
241 The following is copied verbatim from the profile documentation
242 referenced below:
242 referenced below:
243
243
244 When more than one key is provided, additional keys are used as
244 When more than one key is provided, additional keys are used as
245 secondary criteria when the there is equality in all keys selected
245 secondary criteria when the there is equality in all keys selected
246 before them.
246 before them.
247
247
248 Abbreviations can be used for any key names, as long as the
248 Abbreviations can be used for any key names, as long as the
249 abbreviation is unambiguous. The following are the keys currently
249 abbreviation is unambiguous. The following are the keys currently
250 defined:
250 defined:
251
251
252 ============ =====================
252 ============ =====================
253 Valid Arg Meaning
253 Valid Arg Meaning
254 ============ =====================
254 ============ =====================
255 "calls" call count
255 "calls" call count
256 "cumulative" cumulative time
256 "cumulative" cumulative time
257 "file" file name
257 "file" file name
258 "module" file name
258 "module" file name
259 "pcalls" primitive call count
259 "pcalls" primitive call count
260 "line" line number
260 "line" line number
261 "name" function name
261 "name" function name
262 "nfl" name/file/line
262 "nfl" name/file/line
263 "stdname" standard name
263 "stdname" standard name
264 "time" internal time
264 "time" internal time
265 ============ =====================
265 ============ =====================
266
266
267 Note that all sorts on statistics are in descending order (placing
267 Note that all sorts on statistics are in descending order (placing
268 most time consuming items first), where as name, file, and line number
268 most time consuming items first), where as name, file, and line number
269 searches are in ascending order (i.e., alphabetical). The subtle
269 searches are in ascending order (i.e., alphabetical). The subtle
270 distinction between "nfl" and "stdname" is that the standard name is a
270 distinction between "nfl" and "stdname" is that the standard name is a
271 sort of the name as printed, which means that the embedded line
271 sort of the name as printed, which means that the embedded line
272 numbers get compared in an odd way. For example, lines 3, 20, and 40
272 numbers get compared in an odd way. For example, lines 3, 20, and 40
273 would (if the file names were the same) appear in the string order
273 would (if the file names were the same) appear in the string order
274 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
274 "20" "3" and "40". In contrast, "nfl" does a numeric compare of the
275 line numbers. In fact, sort_stats("nfl") is the same as
275 line numbers. In fact, sort_stats("nfl") is the same as
276 sort_stats("name", "file", "line").
276 sort_stats("name", "file", "line").
277
277
278 -T <filename>
278 -T <filename>
279 save profile results as shown on screen to a text
279 save profile results as shown on screen to a text
280 file. The profile is still shown on screen.
280 file. The profile is still shown on screen.
281
281
282 -D <filename>
282 -D <filename>
283 save (via dump_stats) profile statistics to given
283 save (via dump_stats) profile statistics to given
284 filename. This data is in a format understood by the pstats module, and
284 filename. This data is in a format understood by the pstats module, and
285 is generated by a call to the dump_stats() method of profile
285 is generated by a call to the dump_stats() method of profile
286 objects. The profile is still shown on screen.
286 objects. The profile is still shown on screen.
287
287
288 -q
288 -q
289 suppress output to the pager. Best used with -T and/or -D above.
289 suppress output to the pager. Best used with -T and/or -D above.
290
290
291 If you want to run complete programs under the profiler's control, use
291 If you want to run complete programs under the profiler's control, use
292 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
292 ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
293 contains profiler specific options as described here.
293 contains profiler specific options as described here.
294
294
295 You can read the complete documentation for the profile module with::
295 You can read the complete documentation for the profile module with::
296
296
297 In [1]: import profile; profile.help()
297 In [1]: import profile; profile.help()
298
298
299 .. versionchanged:: 7.3
299 .. versionchanged:: 7.3
300 User variables are no longer expanded,
300 User variables are no longer expanded,
301 the magic line is always left unmodified.
301 the magic line is always left unmodified.
302
302
303 """
303 """
304 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
304 opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
305 list_all=True, posix=False)
305 list_all=True, posix=False)
306 if cell is not None:
306 if cell is not None:
307 arg_str += '\n' + cell
307 arg_str += '\n' + cell
308 arg_str = self.shell.transform_cell(arg_str)
308 arg_str = self.shell.transform_cell(arg_str)
309 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
309 return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
310
310
311 def _run_with_profiler(self, code, opts, namespace):
311 def _run_with_profiler(self, code, opts, namespace):
312 """
312 """
313 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
313 Run `code` with profiler. Used by ``%prun`` and ``%run -p``.
314
314
315 Parameters
315 Parameters
316 ----------
316 ----------
317 code : str
317 code : str
318 Code to be executed.
318 Code to be executed.
319 opts : Struct
319 opts : Struct
320 Options parsed by `self.parse_options`.
320 Options parsed by `self.parse_options`.
321 namespace : dict
321 namespace : dict
322 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
322 A dictionary for Python namespace (e.g., `self.shell.user_ns`).
323
323
324 """
324 """
325
325
326 # Fill default values for unspecified options:
326 # Fill default values for unspecified options:
327 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
327 opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
328
328
329 prof = profile.Profile()
329 prof = profile.Profile()
330 try:
330 try:
331 prof = prof.runctx(code, namespace, namespace)
331 prof = prof.runctx(code, namespace, namespace)
332 sys_exit = ''
332 sys_exit = ''
333 except SystemExit:
333 except SystemExit:
334 sys_exit = """*** SystemExit exception caught in code being profiled."""
334 sys_exit = """*** SystemExit exception caught in code being profiled."""
335
335
336 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
336 stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
337
337
338 lims = opts.l
338 lims = opts.l
339 if lims:
339 if lims:
340 lims = [] # rebuild lims with ints/floats/strings
340 lims = [] # rebuild lims with ints/floats/strings
341 for lim in opts.l:
341 for lim in opts.l:
342 try:
342 try:
343 lims.append(int(lim))
343 lims.append(int(lim))
344 except ValueError:
344 except ValueError:
345 try:
345 try:
346 lims.append(float(lim))
346 lims.append(float(lim))
347 except ValueError:
347 except ValueError:
348 lims.append(lim)
348 lims.append(lim)
349
349
350 # Trap output.
350 # Trap output.
351 stdout_trap = StringIO()
351 stdout_trap = StringIO()
352 stats_stream = stats.stream
352 stats_stream = stats.stream
353 try:
353 try:
354 stats.stream = stdout_trap
354 stats.stream = stdout_trap
355 stats.print_stats(*lims)
355 stats.print_stats(*lims)
356 finally:
356 finally:
357 stats.stream = stats_stream
357 stats.stream = stats_stream
358
358
359 output = stdout_trap.getvalue()
359 output = stdout_trap.getvalue()
360 output = output.rstrip()
360 output = output.rstrip()
361
361
362 if 'q' not in opts:
362 if 'q' not in opts:
363 page.page(output)
363 page.page(output)
364 print(sys_exit, end=' ')
364 print(sys_exit, end=' ')
365
365
366 dump_file = opts.D[0]
366 dump_file = opts.D[0]
367 text_file = opts.T[0]
367 text_file = opts.T[0]
368 if dump_file:
368 if dump_file:
369 prof.dump_stats(dump_file)
369 prof.dump_stats(dump_file)
370 print('\n*** Profile stats marshalled to file',\
370 print('\n*** Profile stats marshalled to file',\
371 repr(dump_file)+'.',sys_exit)
371 repr(dump_file)+'.',sys_exit)
372 if text_file:
372 if text_file:
373 pfile = open(text_file,'w')
373 with open(text_file, 'w') as pfile:
374 pfile.write(output)
374 pfile.write(output)
375 pfile.close()
376 print('\n*** Profile printout saved to text file',\
375 print('\n*** Profile printout saved to text file',\
377 repr(text_file)+'.',sys_exit)
376 repr(text_file)+'.',sys_exit)
378
377
379 if 'r' in opts:
378 if 'r' in opts:
380 return stats
379 return stats
381 else:
380 else:
382 return None
381 return None
383
382
384 @line_magic
383 @line_magic
385 def pdb(self, parameter_s=''):
384 def pdb(self, parameter_s=''):
386 """Control the automatic calling of the pdb interactive debugger.
385 """Control the automatic calling of the pdb interactive debugger.
387
386
388 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
387 Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
389 argument it works as a toggle.
388 argument it works as a toggle.
390
389
391 When an exception is triggered, IPython can optionally call the
390 When an exception is triggered, IPython can optionally call the
392 interactive pdb debugger after the traceback printout. %pdb toggles
391 interactive pdb debugger after the traceback printout. %pdb toggles
393 this feature on and off.
392 this feature on and off.
394
393
395 The initial state of this feature is set in your configuration
394 The initial state of this feature is set in your configuration
396 file (the option is ``InteractiveShell.pdb``).
395 file (the option is ``InteractiveShell.pdb``).
397
396
398 If you want to just activate the debugger AFTER an exception has fired,
397 If you want to just activate the debugger AFTER an exception has fired,
399 without having to type '%pdb on' and rerunning your code, you can use
398 without having to type '%pdb on' and rerunning your code, you can use
400 the %debug magic."""
399 the %debug magic."""
401
400
402 par = parameter_s.strip().lower()
401 par = parameter_s.strip().lower()
403
402
404 if par:
403 if par:
405 try:
404 try:
406 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
405 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
407 except KeyError:
406 except KeyError:
408 print ('Incorrect argument. Use on/1, off/0, '
407 print ('Incorrect argument. Use on/1, off/0, '
409 'or nothing for a toggle.')
408 'or nothing for a toggle.')
410 return
409 return
411 else:
410 else:
412 # toggle
411 # toggle
413 new_pdb = not self.shell.call_pdb
412 new_pdb = not self.shell.call_pdb
414
413
415 # set on the shell
414 # set on the shell
416 self.shell.call_pdb = new_pdb
415 self.shell.call_pdb = new_pdb
417 print('Automatic pdb calling has been turned',on_off(new_pdb))
416 print('Automatic pdb calling has been turned',on_off(new_pdb))
418
417
419 @skip_doctest
418 @skip_doctest
420 @magic_arguments.magic_arguments()
419 @magic_arguments.magic_arguments()
421 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
420 @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
422 help="""
421 help="""
423 Set break point at LINE in FILE.
422 Set break point at LINE in FILE.
424 """
423 """
425 )
424 )
426 @magic_arguments.argument('statement', nargs='*',
425 @magic_arguments.argument('statement', nargs='*',
427 help="""
426 help="""
428 Code to run in debugger.
427 Code to run in debugger.
429 You can omit this in cell magic mode.
428 You can omit this in cell magic mode.
430 """
429 """
431 )
430 )
432 @no_var_expand
431 @no_var_expand
433 @line_cell_magic
432 @line_cell_magic
434 def debug(self, line='', cell=None):
433 def debug(self, line='', cell=None):
435 """Activate the interactive debugger.
434 """Activate the interactive debugger.
436
435
437 This magic command support two ways of activating debugger.
436 This magic command support two ways of activating debugger.
438 One is to activate debugger before executing code. This way, you
437 One is to activate debugger before executing code. This way, you
439 can set a break point, to step through the code from the point.
438 can set a break point, to step through the code from the point.
440 You can use this mode by giving statements to execute and optionally
439 You can use this mode by giving statements to execute and optionally
441 a breakpoint.
440 a breakpoint.
442
441
443 The other one is to activate debugger in post-mortem mode. You can
442 The other one is to activate debugger in post-mortem mode. You can
444 activate this mode simply running %debug without any argument.
443 activate this mode simply running %debug without any argument.
445 If an exception has just occurred, this lets you inspect its stack
444 If an exception has just occurred, this lets you inspect its stack
446 frames interactively. Note that this will always work only on the last
445 frames interactively. Note that this will always work only on the last
447 traceback that occurred, so you must call this quickly after an
446 traceback that occurred, so you must call this quickly after an
448 exception that you wish to inspect has fired, because if another one
447 exception that you wish to inspect has fired, because if another one
449 occurs, it clobbers the previous one.
448 occurs, it clobbers the previous one.
450
449
451 If you want IPython to automatically do this on every exception, see
450 If you want IPython to automatically do this on every exception, see
452 the %pdb magic for more details.
451 the %pdb magic for more details.
453
452
454 .. versionchanged:: 7.3
453 .. versionchanged:: 7.3
455 When running code, user variables are no longer expanded,
454 When running code, user variables are no longer expanded,
456 the magic line is always left unmodified.
455 the magic line is always left unmodified.
457
456
458 """
457 """
459 args = magic_arguments.parse_argstring(self.debug, line)
458 args = magic_arguments.parse_argstring(self.debug, line)
460
459
461 if not (args.breakpoint or args.statement or cell):
460 if not (args.breakpoint or args.statement or cell):
462 self._debug_post_mortem()
461 self._debug_post_mortem()
463 else:
462 else:
464 code = "\n".join(args.statement)
463 code = "\n".join(args.statement)
465 if cell:
464 if cell:
466 code += "\n" + cell
465 code += "\n" + cell
467 self._debug_exec(code, args.breakpoint)
466 self._debug_exec(code, args.breakpoint)
468
467
469 def _debug_post_mortem(self):
468 def _debug_post_mortem(self):
470 self.shell.debugger(force=True)
469 self.shell.debugger(force=True)
471
470
472 def _debug_exec(self, code, breakpoint):
471 def _debug_exec(self, code, breakpoint):
473 if breakpoint:
472 if breakpoint:
474 (filename, bp_line) = breakpoint.rsplit(':', 1)
473 (filename, bp_line) = breakpoint.rsplit(':', 1)
475 bp_line = int(bp_line)
474 bp_line = int(bp_line)
476 else:
475 else:
477 (filename, bp_line) = (None, None)
476 (filename, bp_line) = (None, None)
478 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
477 self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
479
478
480 @line_magic
479 @line_magic
481 def tb(self, s):
480 def tb(self, s):
482 """Print the last traceback.
481 """Print the last traceback.
483
482
484 Optionally, specify an exception reporting mode, tuning the
483 Optionally, specify an exception reporting mode, tuning the
485 verbosity of the traceback. By default the currently-active exception
484 verbosity of the traceback. By default the currently-active exception
486 mode is used. See %xmode for changing exception reporting modes.
485 mode is used. See %xmode for changing exception reporting modes.
487
486
488 Valid modes: Plain, Context, Verbose, and Minimal.
487 Valid modes: Plain, Context, Verbose, and Minimal.
489 """
488 """
490 interactive_tb = self.shell.InteractiveTB
489 interactive_tb = self.shell.InteractiveTB
491 if s:
490 if s:
492 # Switch exception reporting mode for this one call.
491 # Switch exception reporting mode for this one call.
493 # Ensure it is switched back.
492 # Ensure it is switched back.
494 def xmode_switch_err(name):
493 def xmode_switch_err(name):
495 warn('Error changing %s exception modes.\n%s' %
494 warn('Error changing %s exception modes.\n%s' %
496 (name,sys.exc_info()[1]))
495 (name,sys.exc_info()[1]))
497
496
498 new_mode = s.strip().capitalize()
497 new_mode = s.strip().capitalize()
499 original_mode = interactive_tb.mode
498 original_mode = interactive_tb.mode
500 try:
499 try:
501 try:
500 try:
502 interactive_tb.set_mode(mode=new_mode)
501 interactive_tb.set_mode(mode=new_mode)
503 except Exception:
502 except Exception:
504 xmode_switch_err('user')
503 xmode_switch_err('user')
505 else:
504 else:
506 self.shell.showtraceback()
505 self.shell.showtraceback()
507 finally:
506 finally:
508 interactive_tb.set_mode(mode=original_mode)
507 interactive_tb.set_mode(mode=original_mode)
509 else:
508 else:
510 self.shell.showtraceback()
509 self.shell.showtraceback()
511
510
512 @skip_doctest
511 @skip_doctest
513 @line_magic
512 @line_magic
514 def run(self, parameter_s='', runner=None,
513 def run(self, parameter_s='', runner=None,
515 file_finder=get_py_filename):
514 file_finder=get_py_filename):
516 """Run the named file inside IPython as a program.
515 """Run the named file inside IPython as a program.
517
516
518 Usage::
517 Usage::
519
518
520 %run [-n -i -e -G]
519 %run [-n -i -e -G]
521 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
520 [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
522 ( -m mod | file ) [args]
521 ( -m mod | file ) [args]
523
522
524 Parameters after the filename are passed as command-line arguments to
523 Parameters after the filename are passed as command-line arguments to
525 the program (put in sys.argv). Then, control returns to IPython's
524 the program (put in sys.argv). Then, control returns to IPython's
526 prompt.
525 prompt.
527
526
528 This is similar to running at a system prompt ``python file args``,
527 This is similar to running at a system prompt ``python file args``,
529 but with the advantage of giving you IPython's tracebacks, and of
528 but with the advantage of giving you IPython's tracebacks, and of
530 loading all variables into your interactive namespace for further use
529 loading all variables into your interactive namespace for further use
531 (unless -p is used, see below).
530 (unless -p is used, see below).
532
531
533 The file is executed in a namespace initially consisting only of
532 The file is executed in a namespace initially consisting only of
534 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
533 ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
535 sees its environment as if it were being run as a stand-alone program
534 sees its environment as if it were being run as a stand-alone program
536 (except for sharing global objects such as previously imported
535 (except for sharing global objects such as previously imported
537 modules). But after execution, the IPython interactive namespace gets
536 modules). But after execution, the IPython interactive namespace gets
538 updated with all variables defined in the program (except for __name__
537 updated with all variables defined in the program (except for __name__
539 and sys.argv). This allows for very convenient loading of code for
538 and sys.argv). This allows for very convenient loading of code for
540 interactive work, while giving each program a 'clean sheet' to run in.
539 interactive work, while giving each program a 'clean sheet' to run in.
541
540
542 Arguments are expanded using shell-like glob match. Patterns
541 Arguments are expanded using shell-like glob match. Patterns
543 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
542 '*', '?', '[seq]' and '[!seq]' can be used. Additionally,
544 tilde '~' will be expanded into user's home directory. Unlike
543 tilde '~' will be expanded into user's home directory. Unlike
545 real shells, quotation does not suppress expansions. Use
544 real shells, quotation does not suppress expansions. Use
546 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
545 *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
547 To completely disable these expansions, you can use -G flag.
546 To completely disable these expansions, you can use -G flag.
548
547
549 On Windows systems, the use of single quotes `'` when specifying
548 On Windows systems, the use of single quotes `'` when specifying
550 a file is not supported. Use double quotes `"`.
549 a file is not supported. Use double quotes `"`.
551
550
552 Options:
551 Options:
553
552
554 -n
553 -n
555 __name__ is NOT set to '__main__', but to the running file's name
554 __name__ is NOT set to '__main__', but to the running file's name
556 without extension (as python does under import). This allows running
555 without extension (as python does under import). This allows running
557 scripts and reloading the definitions in them without calling code
556 scripts and reloading the definitions in them without calling code
558 protected by an ``if __name__ == "__main__"`` clause.
557 protected by an ``if __name__ == "__main__"`` clause.
559
558
560 -i
559 -i
561 run the file in IPython's namespace instead of an empty one. This
560 run the file in IPython's namespace instead of an empty one. This
562 is useful if you are experimenting with code written in a text editor
561 is useful if you are experimenting with code written in a text editor
563 which depends on variables defined interactively.
562 which depends on variables defined interactively.
564
563
565 -e
564 -e
566 ignore sys.exit() calls or SystemExit exceptions in the script
565 ignore sys.exit() calls or SystemExit exceptions in the script
567 being run. This is particularly useful if IPython is being used to
566 being run. This is particularly useful if IPython is being used to
568 run unittests, which always exit with a sys.exit() call. In such
567 run unittests, which always exit with a sys.exit() call. In such
569 cases you are interested in the output of the test results, not in
568 cases you are interested in the output of the test results, not in
570 seeing a traceback of the unittest module.
569 seeing a traceback of the unittest module.
571
570
572 -t
571 -t
573 print timing information at the end of the run. IPython will give
572 print timing information at the end of the run. IPython will give
574 you an estimated CPU time consumption for your script, which under
573 you an estimated CPU time consumption for your script, which under
575 Unix uses the resource module to avoid the wraparound problems of
574 Unix uses the resource module to avoid the wraparound problems of
576 time.clock(). Under Unix, an estimate of time spent on system tasks
575 time.clock(). Under Unix, an estimate of time spent on system tasks
577 is also given (for Windows platforms this is reported as 0.0).
576 is also given (for Windows platforms this is reported as 0.0).
578
577
579 If -t is given, an additional ``-N<N>`` option can be given, where <N>
578 If -t is given, an additional ``-N<N>`` option can be given, where <N>
580 must be an integer indicating how many times you want the script to
579 must be an integer indicating how many times you want the script to
581 run. The final timing report will include total and per run results.
580 run. The final timing report will include total and per run results.
582
581
583 For example (testing the script uniq_stable.py)::
582 For example (testing the script uniq_stable.py)::
584
583
585 In [1]: run -t uniq_stable
584 In [1]: run -t uniq_stable
586
585
587 IPython CPU timings (estimated):
586 IPython CPU timings (estimated):
588 User : 0.19597 s.
587 User : 0.19597 s.
589 System: 0.0 s.
588 System: 0.0 s.
590
589
591 In [2]: run -t -N5 uniq_stable
590 In [2]: run -t -N5 uniq_stable
592
591
593 IPython CPU timings (estimated):
592 IPython CPU timings (estimated):
594 Total runs performed: 5
593 Total runs performed: 5
595 Times : Total Per run
594 Times : Total Per run
596 User : 0.910862 s, 0.1821724 s.
595 User : 0.910862 s, 0.1821724 s.
597 System: 0.0 s, 0.0 s.
596 System: 0.0 s, 0.0 s.
598
597
599 -d
598 -d
600 run your program under the control of pdb, the Python debugger.
599 run your program under the control of pdb, the Python debugger.
601 This allows you to execute your program step by step, watch variables,
600 This allows you to execute your program step by step, watch variables,
602 etc. Internally, what IPython does is similar to calling::
601 etc. Internally, what IPython does is similar to calling::
603
602
604 pdb.run('execfile("YOURFILENAME")')
603 pdb.run('execfile("YOURFILENAME")')
605
604
606 with a breakpoint set on line 1 of your file. You can change the line
605 with a breakpoint set on line 1 of your file. You can change the line
607 number for this automatic breakpoint to be <N> by using the -bN option
606 number for this automatic breakpoint to be <N> by using the -bN option
608 (where N must be an integer). For example::
607 (where N must be an integer). For example::
609
608
610 %run -d -b40 myscript
609 %run -d -b40 myscript
611
610
612 will set the first breakpoint at line 40 in myscript.py. Note that
611 will set the first breakpoint at line 40 in myscript.py. Note that
613 the first breakpoint must be set on a line which actually does
612 the first breakpoint must be set on a line which actually does
614 something (not a comment or docstring) for it to stop execution.
613 something (not a comment or docstring) for it to stop execution.
615
614
616 Or you can specify a breakpoint in a different file::
615 Or you can specify a breakpoint in a different file::
617
616
618 %run -d -b myotherfile.py:20 myscript
617 %run -d -b myotherfile.py:20 myscript
619
618
620 When the pdb debugger starts, you will see a (Pdb) prompt. You must
619 When the pdb debugger starts, you will see a (Pdb) prompt. You must
621 first enter 'c' (without quotes) to start execution up to the first
620 first enter 'c' (without quotes) to start execution up to the first
622 breakpoint.
621 breakpoint.
623
622
624 Entering 'help' gives information about the use of the debugger. You
623 Entering 'help' gives information about the use of the debugger. You
625 can easily see pdb's full documentation with "import pdb;pdb.help()"
624 can easily see pdb's full documentation with "import pdb;pdb.help()"
626 at a prompt.
625 at a prompt.
627
626
628 -p
627 -p
629 run program under the control of the Python profiler module (which
628 run program under the control of the Python profiler module (which
630 prints a detailed report of execution times, function calls, etc).
629 prints a detailed report of execution times, function calls, etc).
631
630
632 You can pass other options after -p which affect the behavior of the
631 You can pass other options after -p which affect the behavior of the
633 profiler itself. See the docs for %prun for details.
632 profiler itself. See the docs for %prun for details.
634
633
635 In this mode, the program's variables do NOT propagate back to the
634 In this mode, the program's variables do NOT propagate back to the
636 IPython interactive namespace (because they remain in the namespace
635 IPython interactive namespace (because they remain in the namespace
637 where the profiler executes them).
636 where the profiler executes them).
638
637
639 Internally this triggers a call to %prun, see its documentation for
638 Internally this triggers a call to %prun, see its documentation for
640 details on the options available specifically for profiling.
639 details on the options available specifically for profiling.
641
640
642 There is one special usage for which the text above doesn't apply:
641 There is one special usage for which the text above doesn't apply:
643 if the filename ends with .ipy[nb], the file is run as ipython script,
642 if the filename ends with .ipy[nb], the file is run as ipython script,
644 just as if the commands were written on IPython prompt.
643 just as if the commands were written on IPython prompt.
645
644
646 -m
645 -m
647 specify module name to load instead of script path. Similar to
646 specify module name to load instead of script path. Similar to
648 the -m option for the python interpreter. Use this option last if you
647 the -m option for the python interpreter. Use this option last if you
649 want to combine with other %run options. Unlike the python interpreter
648 want to combine with other %run options. Unlike the python interpreter
650 only source modules are allowed no .pyc or .pyo files.
649 only source modules are allowed no .pyc or .pyo files.
651 For example::
650 For example::
652
651
653 %run -m example
652 %run -m example
654
653
655 will run the example module.
654 will run the example module.
656
655
657 -G
656 -G
658 disable shell-like glob expansion of arguments.
657 disable shell-like glob expansion of arguments.
659
658
660 """
659 """
661
660
662 # Logic to handle issue #3664
661 # Logic to handle issue #3664
663 # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
662 # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
664 if '-m' in parameter_s and '--' not in parameter_s:
663 if '-m' in parameter_s and '--' not in parameter_s:
665 argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
664 argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
666 for idx, arg in enumerate(argv):
665 for idx, arg in enumerate(argv):
667 if arg and arg.startswith('-') and arg != '-':
666 if arg and arg.startswith('-') and arg != '-':
668 if arg == '-m':
667 if arg == '-m':
669 argv.insert(idx + 2, '--')
668 argv.insert(idx + 2, '--')
670 break
669 break
671 else:
670 else:
672 # Positional arg, break
671 # Positional arg, break
673 break
672 break
674 parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
673 parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
675
674
676 # get arguments and set sys.argv for program to be run.
675 # get arguments and set sys.argv for program to be run.
677 opts, arg_lst = self.parse_options(parameter_s,
676 opts, arg_lst = self.parse_options(parameter_s,
678 'nidtN:b:pD:l:rs:T:em:G',
677 'nidtN:b:pD:l:rs:T:em:G',
679 mode='list', list_all=1)
678 mode='list', list_all=1)
680 if "m" in opts:
679 if "m" in opts:
681 modulename = opts["m"][0]
680 modulename = opts["m"][0]
682 modpath = find_mod(modulename)
681 modpath = find_mod(modulename)
683 if modpath is None:
682 if modpath is None:
684 warn('%r is not a valid modulename on sys.path'%modulename)
683 warn('%r is not a valid modulename on sys.path'%modulename)
685 return
684 return
686 arg_lst = [modpath] + arg_lst
685 arg_lst = [modpath] + arg_lst
687 try:
686 try:
688 fpath = None # initialize to make sure fpath is in scope later
687 fpath = None # initialize to make sure fpath is in scope later
689 fpath = arg_lst[0]
688 fpath = arg_lst[0]
690 filename = file_finder(fpath)
689 filename = file_finder(fpath)
691 except IndexError:
690 except IndexError:
692 warn('you must provide at least a filename.')
691 warn('you must provide at least a filename.')
693 print('\n%run:\n', oinspect.getdoc(self.run))
692 print('\n%run:\n', oinspect.getdoc(self.run))
694 return
693 return
695 except IOError as e:
694 except IOError as e:
696 try:
695 try:
697 msg = str(e)
696 msg = str(e)
698 except UnicodeError:
697 except UnicodeError:
699 msg = e.message
698 msg = e.message
700 if os.name == 'nt' and re.match(r"^'.*'$",fpath):
699 if os.name == 'nt' and re.match(r"^'.*'$",fpath):
701 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
700 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
702 error(msg)
701 error(msg)
703 return
702 return
704
703
705 if filename.lower().endswith(('.ipy', '.ipynb')):
704 if filename.lower().endswith(('.ipy', '.ipynb')):
706 with preserve_keys(self.shell.user_ns, '__file__'):
705 with preserve_keys(self.shell.user_ns, '__file__'):
707 self.shell.user_ns['__file__'] = filename
706 self.shell.user_ns['__file__'] = filename
708 self.shell.safe_execfile_ipy(filename)
707 self.shell.safe_execfile_ipy(filename)
709 return
708 return
710
709
711 # Control the response to exit() calls made by the script being run
710 # Control the response to exit() calls made by the script being run
712 exit_ignore = 'e' in opts
711 exit_ignore = 'e' in opts
713
712
714 # Make sure that the running script gets a proper sys.argv as if it
713 # Make sure that the running script gets a proper sys.argv as if it
715 # were run from a system shell.
714 # were run from a system shell.
716 save_argv = sys.argv # save it for later restoring
715 save_argv = sys.argv # save it for later restoring
717
716
718 if 'G' in opts:
717 if 'G' in opts:
719 args = arg_lst[1:]
718 args = arg_lst[1:]
720 else:
719 else:
721 # tilde and glob expansion
720 # tilde and glob expansion
722 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
721 args = shellglob(map(os.path.expanduser, arg_lst[1:]))
723
722
724 sys.argv = [filename] + args # put in the proper filename
723 sys.argv = [filename] + args # put in the proper filename
725
724
726 if 'n' in opts:
725 if 'n' in opts:
727 name = os.path.splitext(os.path.basename(filename))[0]
726 name = os.path.splitext(os.path.basename(filename))[0]
728 else:
727 else:
729 name = '__main__'
728 name = '__main__'
730
729
731 if 'i' in opts:
730 if 'i' in opts:
732 # Run in user's interactive namespace
731 # Run in user's interactive namespace
733 prog_ns = self.shell.user_ns
732 prog_ns = self.shell.user_ns
734 __name__save = self.shell.user_ns['__name__']
733 __name__save = self.shell.user_ns['__name__']
735 prog_ns['__name__'] = name
734 prog_ns['__name__'] = name
736 main_mod = self.shell.user_module
735 main_mod = self.shell.user_module
737
736
738 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
737 # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
739 # set the __file__ global in the script's namespace
738 # set the __file__ global in the script's namespace
740 # TK: Is this necessary in interactive mode?
739 # TK: Is this necessary in interactive mode?
741 prog_ns['__file__'] = filename
740 prog_ns['__file__'] = filename
742 else:
741 else:
743 # Run in a fresh, empty namespace
742 # Run in a fresh, empty namespace
744
743
745 # The shell MUST hold a reference to prog_ns so after %run
744 # The shell MUST hold a reference to prog_ns so after %run
746 # exits, the python deletion mechanism doesn't zero it out
745 # exits, the python deletion mechanism doesn't zero it out
747 # (leaving dangling references). See interactiveshell for details
746 # (leaving dangling references). See interactiveshell for details
748 main_mod = self.shell.new_main_mod(filename, name)
747 main_mod = self.shell.new_main_mod(filename, name)
749 prog_ns = main_mod.__dict__
748 prog_ns = main_mod.__dict__
750
749
751 # pickle fix. See interactiveshell for an explanation. But we need to
750 # pickle fix. See interactiveshell for an explanation. But we need to
752 # make sure that, if we overwrite __main__, we replace it at the end
751 # make sure that, if we overwrite __main__, we replace it at the end
753 main_mod_name = prog_ns['__name__']
752 main_mod_name = prog_ns['__name__']
754
753
755 if main_mod_name == '__main__':
754 if main_mod_name == '__main__':
756 restore_main = sys.modules['__main__']
755 restore_main = sys.modules['__main__']
757 else:
756 else:
758 restore_main = False
757 restore_main = False
759
758
760 # This needs to be undone at the end to prevent holding references to
759 # This needs to be undone at the end to prevent holding references to
761 # every single object ever created.
760 # every single object ever created.
762 sys.modules[main_mod_name] = main_mod
761 sys.modules[main_mod_name] = main_mod
763
762
764 if 'p' in opts or 'd' in opts:
763 if 'p' in opts or 'd' in opts:
765 if 'm' in opts:
764 if 'm' in opts:
766 code = 'run_module(modulename, prog_ns)'
765 code = 'run_module(modulename, prog_ns)'
767 code_ns = {
766 code_ns = {
768 'run_module': self.shell.safe_run_module,
767 'run_module': self.shell.safe_run_module,
769 'prog_ns': prog_ns,
768 'prog_ns': prog_ns,
770 'modulename': modulename,
769 'modulename': modulename,
771 }
770 }
772 else:
771 else:
773 if 'd' in opts:
772 if 'd' in opts:
774 # allow exceptions to raise in debug mode
773 # allow exceptions to raise in debug mode
775 code = 'execfile(filename, prog_ns, raise_exceptions=True)'
774 code = 'execfile(filename, prog_ns, raise_exceptions=True)'
776 else:
775 else:
777 code = 'execfile(filename, prog_ns)'
776 code = 'execfile(filename, prog_ns)'
778 code_ns = {
777 code_ns = {
779 'execfile': self.shell.safe_execfile,
778 'execfile': self.shell.safe_execfile,
780 'prog_ns': prog_ns,
779 'prog_ns': prog_ns,
781 'filename': get_py_filename(filename),
780 'filename': get_py_filename(filename),
782 }
781 }
783
782
784 try:
783 try:
785 stats = None
784 stats = None
786 if 'p' in opts:
785 if 'p' in opts:
787 stats = self._run_with_profiler(code, opts, code_ns)
786 stats = self._run_with_profiler(code, opts, code_ns)
788 else:
787 else:
789 if 'd' in opts:
788 if 'd' in opts:
790 bp_file, bp_line = parse_breakpoint(
789 bp_file, bp_line = parse_breakpoint(
791 opts.get('b', ['1'])[0], filename)
790 opts.get('b', ['1'])[0], filename)
792 self._run_with_debugger(
791 self._run_with_debugger(
793 code, code_ns, filename, bp_line, bp_file)
792 code, code_ns, filename, bp_line, bp_file)
794 else:
793 else:
795 if 'm' in opts:
794 if 'm' in opts:
796 def run():
795 def run():
797 self.shell.safe_run_module(modulename, prog_ns)
796 self.shell.safe_run_module(modulename, prog_ns)
798 else:
797 else:
799 if runner is None:
798 if runner is None:
800 runner = self.default_runner
799 runner = self.default_runner
801 if runner is None:
800 if runner is None:
802 runner = self.shell.safe_execfile
801 runner = self.shell.safe_execfile
803
802
804 def run():
803 def run():
805 runner(filename, prog_ns, prog_ns,
804 runner(filename, prog_ns, prog_ns,
806 exit_ignore=exit_ignore)
805 exit_ignore=exit_ignore)
807
806
808 if 't' in opts:
807 if 't' in opts:
809 # timed execution
808 # timed execution
810 try:
809 try:
811 nruns = int(opts['N'][0])
810 nruns = int(opts['N'][0])
812 if nruns < 1:
811 if nruns < 1:
813 error('Number of runs must be >=1')
812 error('Number of runs must be >=1')
814 return
813 return
815 except (KeyError):
814 except (KeyError):
816 nruns = 1
815 nruns = 1
817 self._run_with_timing(run, nruns)
816 self._run_with_timing(run, nruns)
818 else:
817 else:
819 # regular execution
818 # regular execution
820 run()
819 run()
821
820
822 if 'i' in opts:
821 if 'i' in opts:
823 self.shell.user_ns['__name__'] = __name__save
822 self.shell.user_ns['__name__'] = __name__save
824 else:
823 else:
825 # update IPython interactive namespace
824 # update IPython interactive namespace
826
825
827 # Some forms of read errors on the file may mean the
826 # Some forms of read errors on the file may mean the
828 # __name__ key was never set; using pop we don't have to
827 # __name__ key was never set; using pop we don't have to
829 # worry about a possible KeyError.
828 # worry about a possible KeyError.
830 prog_ns.pop('__name__', None)
829 prog_ns.pop('__name__', None)
831
830
832 with preserve_keys(self.shell.user_ns, '__file__'):
831 with preserve_keys(self.shell.user_ns, '__file__'):
833 self.shell.user_ns.update(prog_ns)
832 self.shell.user_ns.update(prog_ns)
834 finally:
833 finally:
835 # It's a bit of a mystery why, but __builtins__ can change from
834 # It's a bit of a mystery why, but __builtins__ can change from
836 # being a module to becoming a dict missing some key data after
835 # being a module to becoming a dict missing some key data after
837 # %run. As best I can see, this is NOT something IPython is doing
836 # %run. As best I can see, this is NOT something IPython is doing
838 # at all, and similar problems have been reported before:
837 # at all, and similar problems have been reported before:
839 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
838 # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
840 # Since this seems to be done by the interpreter itself, the best
839 # Since this seems to be done by the interpreter itself, the best
841 # we can do is to at least restore __builtins__ for the user on
840 # we can do is to at least restore __builtins__ for the user on
842 # exit.
841 # exit.
843 self.shell.user_ns['__builtins__'] = builtin_mod
842 self.shell.user_ns['__builtins__'] = builtin_mod
844
843
845 # Ensure key global structures are restored
844 # Ensure key global structures are restored
846 sys.argv = save_argv
845 sys.argv = save_argv
847 if restore_main:
846 if restore_main:
848 sys.modules['__main__'] = restore_main
847 sys.modules['__main__'] = restore_main
849 else:
848 else:
850 # Remove from sys.modules the reference to main_mod we'd
849 # Remove from sys.modules the reference to main_mod we'd
851 # added. Otherwise it will trap references to objects
850 # added. Otherwise it will trap references to objects
852 # contained therein.
851 # contained therein.
853 del sys.modules[main_mod_name]
852 del sys.modules[main_mod_name]
854
853
855 return stats
854 return stats
856
855
857 def _run_with_debugger(self, code, code_ns, filename=None,
856 def _run_with_debugger(self, code, code_ns, filename=None,
858 bp_line=None, bp_file=None):
857 bp_line=None, bp_file=None):
859 """
858 """
860 Run `code` in debugger with a break point.
859 Run `code` in debugger with a break point.
861
860
862 Parameters
861 Parameters
863 ----------
862 ----------
864 code : str
863 code : str
865 Code to execute.
864 Code to execute.
866 code_ns : dict
865 code_ns : dict
867 A namespace in which `code` is executed.
866 A namespace in which `code` is executed.
868 filename : str
867 filename : str
869 `code` is ran as if it is in `filename`.
868 `code` is ran as if it is in `filename`.
870 bp_line : int, optional
869 bp_line : int, optional
871 Line number of the break point.
870 Line number of the break point.
872 bp_file : str, optional
871 bp_file : str, optional
873 Path to the file in which break point is specified.
872 Path to the file in which break point is specified.
874 `filename` is used if not given.
873 `filename` is used if not given.
875
874
876 Raises
875 Raises
877 ------
876 ------
878 UsageError
877 UsageError
879 If the break point given by `bp_line` is not valid.
878 If the break point given by `bp_line` is not valid.
880
879
881 """
880 """
882 deb = self.shell.InteractiveTB.pdb
881 deb = self.shell.InteractiveTB.pdb
883 if not deb:
882 if not deb:
884 self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
883 self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
885 deb = self.shell.InteractiveTB.pdb
884 deb = self.shell.InteractiveTB.pdb
886
885
887 # deb.checkline() fails if deb.curframe exists but is None; it can
886 # deb.checkline() fails if deb.curframe exists but is None; it can
888 # handle it not existing. https://github.com/ipython/ipython/issues/10028
887 # handle it not existing. https://github.com/ipython/ipython/issues/10028
889 if hasattr(deb, 'curframe'):
888 if hasattr(deb, 'curframe'):
890 del deb.curframe
889 del deb.curframe
891
890
892 # reset Breakpoint state, which is moronically kept
891 # reset Breakpoint state, which is moronically kept
893 # in a class
892 # in a class
894 bdb.Breakpoint.next = 1
893 bdb.Breakpoint.next = 1
895 bdb.Breakpoint.bplist = {}
894 bdb.Breakpoint.bplist = {}
896 bdb.Breakpoint.bpbynumber = [None]
895 bdb.Breakpoint.bpbynumber = [None]
897 deb.clear_all_breaks()
896 deb.clear_all_breaks()
898 if bp_line is not None:
897 if bp_line is not None:
899 # Set an initial breakpoint to stop execution
898 # Set an initial breakpoint to stop execution
900 maxtries = 10
899 maxtries = 10
901 bp_file = bp_file or filename
900 bp_file = bp_file or filename
902 checkline = deb.checkline(bp_file, bp_line)
901 checkline = deb.checkline(bp_file, bp_line)
903 if not checkline:
902 if not checkline:
904 for bp in range(bp_line + 1, bp_line + maxtries + 1):
903 for bp in range(bp_line + 1, bp_line + maxtries + 1):
905 if deb.checkline(bp_file, bp):
904 if deb.checkline(bp_file, bp):
906 break
905 break
907 else:
906 else:
908 msg = ("\nI failed to find a valid line to set "
907 msg = ("\nI failed to find a valid line to set "
909 "a breakpoint\n"
908 "a breakpoint\n"
910 "after trying up to line: %s.\n"
909 "after trying up to line: %s.\n"
911 "Please set a valid breakpoint manually "
910 "Please set a valid breakpoint manually "
912 "with the -b option." % bp)
911 "with the -b option." % bp)
913 raise UsageError(msg)
912 raise UsageError(msg)
914 # if we find a good linenumber, set the breakpoint
913 # if we find a good linenumber, set the breakpoint
915 deb.do_break('%s:%s' % (bp_file, bp_line))
914 deb.do_break('%s:%s' % (bp_file, bp_line))
916
915
917 if filename:
916 if filename:
918 # Mimic Pdb._runscript(...)
917 # Mimic Pdb._runscript(...)
919 deb._wait_for_mainpyfile = True
918 deb._wait_for_mainpyfile = True
920 deb.mainpyfile = deb.canonic(filename)
919 deb.mainpyfile = deb.canonic(filename)
921
920
922 # Start file run
921 # Start file run
923 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
922 print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
924 try:
923 try:
925 if filename:
924 if filename:
926 # save filename so it can be used by methods on the deb object
925 # save filename so it can be used by methods on the deb object
927 deb._exec_filename = filename
926 deb._exec_filename = filename
928 while True:
927 while True:
929 try:
928 try:
930 deb.run(code, code_ns)
929 deb.run(code, code_ns)
931 except Restart:
930 except Restart:
932 print("Restarting")
931 print("Restarting")
933 if filename:
932 if filename:
934 deb._wait_for_mainpyfile = True
933 deb._wait_for_mainpyfile = True
935 deb.mainpyfile = deb.canonic(filename)
934 deb.mainpyfile = deb.canonic(filename)
936 continue
935 continue
937 else:
936 else:
938 break
937 break
939
938
940
939
941 except:
940 except:
942 etype, value, tb = sys.exc_info()
941 etype, value, tb = sys.exc_info()
943 # Skip three frames in the traceback: the %run one,
942 # Skip three frames in the traceback: the %run one,
944 # one inside bdb.py, and the command-line typed by the
943 # one inside bdb.py, and the command-line typed by the
945 # user (run by exec in pdb itself).
944 # user (run by exec in pdb itself).
946 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
945 self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
947
946
948 @staticmethod
947 @staticmethod
949 def _run_with_timing(run, nruns):
948 def _run_with_timing(run, nruns):
950 """
949 """
951 Run function `run` and print timing information.
950 Run function `run` and print timing information.
952
951
953 Parameters
952 Parameters
954 ----------
953 ----------
955 run : callable
954 run : callable
956 Any callable object which takes no argument.
955 Any callable object which takes no argument.
957 nruns : int
956 nruns : int
958 Number of times to execute `run`.
957 Number of times to execute `run`.
959
958
960 """
959 """
961 twall0 = time.perf_counter()
960 twall0 = time.perf_counter()
962 if nruns == 1:
961 if nruns == 1:
963 t0 = clock2()
962 t0 = clock2()
964 run()
963 run()
965 t1 = clock2()
964 t1 = clock2()
966 t_usr = t1[0] - t0[0]
965 t_usr = t1[0] - t0[0]
967 t_sys = t1[1] - t0[1]
966 t_sys = t1[1] - t0[1]
968 print("\nIPython CPU timings (estimated):")
967 print("\nIPython CPU timings (estimated):")
969 print(" User : %10.2f s." % t_usr)
968 print(" User : %10.2f s." % t_usr)
970 print(" System : %10.2f s." % t_sys)
969 print(" System : %10.2f s." % t_sys)
971 else:
970 else:
972 runs = range(nruns)
971 runs = range(nruns)
973 t0 = clock2()
972 t0 = clock2()
974 for nr in runs:
973 for nr in runs:
975 run()
974 run()
976 t1 = clock2()
975 t1 = clock2()
977 t_usr = t1[0] - t0[0]
976 t_usr = t1[0] - t0[0]
978 t_sys = t1[1] - t0[1]
977 t_sys = t1[1] - t0[1]
979 print("\nIPython CPU timings (estimated):")
978 print("\nIPython CPU timings (estimated):")
980 print("Total runs performed:", nruns)
979 print("Total runs performed:", nruns)
981 print(" Times : %10s %10s" % ('Total', 'Per run'))
980 print(" Times : %10s %10s" % ('Total', 'Per run'))
982 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
981 print(" User : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
983 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
982 print(" System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
984 twall1 = time.perf_counter()
983 twall1 = time.perf_counter()
985 print("Wall time: %10.2f s." % (twall1 - twall0))
984 print("Wall time: %10.2f s." % (twall1 - twall0))
986
985
987 @skip_doctest
986 @skip_doctest
988 @no_var_expand
987 @no_var_expand
989 @line_cell_magic
988 @line_cell_magic
990 @needs_local_scope
989 @needs_local_scope
991 def timeit(self, line='', cell=None, local_ns=None):
990 def timeit(self, line='', cell=None, local_ns=None):
992 """Time execution of a Python statement or expression
991 """Time execution of a Python statement or expression
993
992
994 Usage, in line mode:
993 Usage, in line mode:
995 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
994 %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
996 or in cell mode:
995 or in cell mode:
997 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
996 %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
998 code
997 code
999 code...
998 code...
1000
999
1001 Time execution of a Python statement or expression using the timeit
1000 Time execution of a Python statement or expression using the timeit
1002 module. This function can be used both as a line and cell magic:
1001 module. This function can be used both as a line and cell magic:
1003
1002
1004 - In line mode you can time a single-line statement (though multiple
1003 - In line mode you can time a single-line statement (though multiple
1005 ones can be chained with using semicolons).
1004 ones can be chained with using semicolons).
1006
1005
1007 - In cell mode, the statement in the first line is used as setup code
1006 - In cell mode, the statement in the first line is used as setup code
1008 (executed but not timed) and the body of the cell is timed. The cell
1007 (executed but not timed) and the body of the cell is timed. The cell
1009 body has access to any variables created in the setup code.
1008 body has access to any variables created in the setup code.
1010
1009
1011 Options:
1010 Options:
1012 -n<N>: execute the given statement <N> times in a loop. If <N> is not
1011 -n<N>: execute the given statement <N> times in a loop. If <N> is not
1013 provided, <N> is determined so as to get sufficient accuracy.
1012 provided, <N> is determined so as to get sufficient accuracy.
1014
1013
1015 -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
1014 -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
1016 best result.
1015 best result.
1017 Default: 7
1016 Default: 7
1018
1017
1019 -t: use time.time to measure the time, which is the default on Unix.
1018 -t: use time.time to measure the time, which is the default on Unix.
1020 This function measures wall time.
1019 This function measures wall time.
1021
1020
1022 -c: use time.clock to measure the time, which is the default on
1021 -c: use time.clock to measure the time, which is the default on
1023 Windows and measures wall time. On Unix, resource.getrusage is used
1022 Windows and measures wall time. On Unix, resource.getrusage is used
1024 instead and returns the CPU user time.
1023 instead and returns the CPU user time.
1025
1024
1026 -p<P>: use a precision of <P> digits to display the timing result.
1025 -p<P>: use a precision of <P> digits to display the timing result.
1027 Default: 3
1026 Default: 3
1028
1027
1029 -q: Quiet, do not print result.
1028 -q: Quiet, do not print result.
1030
1029
1031 -o: return a TimeitResult that can be stored in a variable to inspect
1030 -o: return a TimeitResult that can be stored in a variable to inspect
1032 the result in more details.
1031 the result in more details.
1033
1032
1034 .. versionchanged:: 7.3
1033 .. versionchanged:: 7.3
1035 User variables are no longer expanded,
1034 User variables are no longer expanded,
1036 the magic line is always left unmodified.
1035 the magic line is always left unmodified.
1037
1036
1038 Examples
1037 Examples
1039 --------
1038 --------
1040 ::
1039 ::
1041
1040
1042 In [1]: %timeit pass
1041 In [1]: %timeit pass
1043 8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
1042 8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
1044
1043
1045 In [2]: u = None
1044 In [2]: u = None
1046
1045
1047 In [3]: %timeit u is None
1046 In [3]: %timeit u is None
1048 29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
1047 29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
1049
1048
1050 In [4]: %timeit -r 4 u == None
1049 In [4]: %timeit -r 4 u == None
1051
1050
1052 In [5]: import time
1051 In [5]: import time
1053
1052
1054 In [6]: %timeit -n1 time.sleep(2)
1053 In [6]: %timeit -n1 time.sleep(2)
1055
1054
1056
1055
1057 The times reported by %timeit will be slightly higher than those
1056 The times reported by %timeit will be slightly higher than those
1058 reported by the timeit.py script when variables are accessed. This is
1057 reported by the timeit.py script when variables are accessed. This is
1059 due to the fact that %timeit executes the statement in the namespace
1058 due to the fact that %timeit executes the statement in the namespace
1060 of the shell, compared with timeit.py, which uses a single setup
1059 of the shell, compared with timeit.py, which uses a single setup
1061 statement to import function or create variables. Generally, the bias
1060 statement to import function or create variables. Generally, the bias
1062 does not matter as long as results from timeit.py are not mixed with
1061 does not matter as long as results from timeit.py are not mixed with
1063 those from %timeit."""
1062 those from %timeit."""
1064
1063
1065 opts, stmt = self.parse_options(line,'n:r:tcp:qo',
1064 opts, stmt = self.parse_options(line,'n:r:tcp:qo',
1066 posix=False, strict=False)
1065 posix=False, strict=False)
1067 if stmt == "" and cell is None:
1066 if stmt == "" and cell is None:
1068 return
1067 return
1069
1068
1070 timefunc = timeit.default_timer
1069 timefunc = timeit.default_timer
1071 number = int(getattr(opts, "n", 0))
1070 number = int(getattr(opts, "n", 0))
1072 default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
1071 default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
1073 repeat = int(getattr(opts, "r", default_repeat))
1072 repeat = int(getattr(opts, "r", default_repeat))
1074 precision = int(getattr(opts, "p", 3))
1073 precision = int(getattr(opts, "p", 3))
1075 quiet = 'q' in opts
1074 quiet = 'q' in opts
1076 return_result = 'o' in opts
1075 return_result = 'o' in opts
1077 if hasattr(opts, "t"):
1076 if hasattr(opts, "t"):
1078 timefunc = time.time
1077 timefunc = time.time
1079 if hasattr(opts, "c"):
1078 if hasattr(opts, "c"):
1080 timefunc = clock
1079 timefunc = clock
1081
1080
1082 timer = Timer(timer=timefunc)
1081 timer = Timer(timer=timefunc)
1083 # this code has tight coupling to the inner workings of timeit.Timer,
1082 # this code has tight coupling to the inner workings of timeit.Timer,
1084 # but is there a better way to achieve that the code stmt has access
1083 # but is there a better way to achieve that the code stmt has access
1085 # to the shell namespace?
1084 # to the shell namespace?
1086 transform = self.shell.transform_cell
1085 transform = self.shell.transform_cell
1087
1086
1088 if cell is None:
1087 if cell is None:
1089 # called as line magic
1088 # called as line magic
1090 ast_setup = self.shell.compile.ast_parse("pass")
1089 ast_setup = self.shell.compile.ast_parse("pass")
1091 ast_stmt = self.shell.compile.ast_parse(transform(stmt))
1090 ast_stmt = self.shell.compile.ast_parse(transform(stmt))
1092 else:
1091 else:
1093 ast_setup = self.shell.compile.ast_parse(transform(stmt))
1092 ast_setup = self.shell.compile.ast_parse(transform(stmt))
1094 ast_stmt = self.shell.compile.ast_parse(transform(cell))
1093 ast_stmt = self.shell.compile.ast_parse(transform(cell))
1095
1094
1096 ast_setup = self.shell.transform_ast(ast_setup)
1095 ast_setup = self.shell.transform_ast(ast_setup)
1097 ast_stmt = self.shell.transform_ast(ast_stmt)
1096 ast_stmt = self.shell.transform_ast(ast_stmt)
1098
1097
1099 # Check that these compile to valid Python code *outside* the timer func
1098 # Check that these compile to valid Python code *outside* the timer func
1100 # Invalid code may become valid when put inside the function & loop,
1099 # Invalid code may become valid when put inside the function & loop,
1101 # which messes up error messages.
1100 # which messes up error messages.
1102 # https://github.com/ipython/ipython/issues/10636
1101 # https://github.com/ipython/ipython/issues/10636
1103 self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
1102 self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
1104 self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
1103 self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
1105
1104
1106 # This codestring is taken from timeit.template - we fill it in as an
1105 # This codestring is taken from timeit.template - we fill it in as an
1107 # AST, so that we can apply our AST transformations to the user code
1106 # AST, so that we can apply our AST transformations to the user code
1108 # without affecting the timing code.
1107 # without affecting the timing code.
1109 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
1108 timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
1110 ' setup\n'
1109 ' setup\n'
1111 ' _t0 = _timer()\n'
1110 ' _t0 = _timer()\n'
1112 ' for _i in _it:\n'
1111 ' for _i in _it:\n'
1113 ' stmt\n'
1112 ' stmt\n'
1114 ' _t1 = _timer()\n'
1113 ' _t1 = _timer()\n'
1115 ' return _t1 - _t0\n')
1114 ' return _t1 - _t0\n')
1116
1115
1117 timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
1116 timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
1118 timeit_ast = ast.fix_missing_locations(timeit_ast)
1117 timeit_ast = ast.fix_missing_locations(timeit_ast)
1119
1118
1120 # Track compilation time so it can be reported if too long
1119 # Track compilation time so it can be reported if too long
1121 # Minimum time above which compilation time will be reported
1120 # Minimum time above which compilation time will be reported
1122 tc_min = 0.1
1121 tc_min = 0.1
1123
1122
1124 t0 = clock()
1123 t0 = clock()
1125 code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
1124 code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
1126 tc = clock()-t0
1125 tc = clock()-t0
1127
1126
1128 ns = {}
1127 ns = {}
1129 glob = self.shell.user_ns
1128 glob = self.shell.user_ns
1130 # handles global vars with same name as local vars. We store them in conflict_globs.
1129 # handles global vars with same name as local vars. We store them in conflict_globs.
1131 if local_ns is not None:
1130 if local_ns is not None:
1132 conflict_globs = {}
1131 conflict_globs = {}
1133 for var_name, var_val in glob.items():
1132 for var_name, var_val in glob.items():
1134 if var_name in local_ns:
1133 if var_name in local_ns:
1135 conflict_globs[var_name] = var_val
1134 conflict_globs[var_name] = var_val
1136 glob.update(local_ns)
1135 glob.update(local_ns)
1137
1136
1138 exec(code, glob, ns)
1137 exec(code, glob, ns)
1139 timer.inner = ns["inner"]
1138 timer.inner = ns["inner"]
1140
1139
1141 # This is used to check if there is a huge difference between the
1140 # This is used to check if there is a huge difference between the
1142 # best and worst timings.
1141 # best and worst timings.
1143 # Issue: https://github.com/ipython/ipython/issues/6471
1142 # Issue: https://github.com/ipython/ipython/issues/6471
1144 if number == 0:
1143 if number == 0:
1145 # determine number so that 0.2 <= total time < 2.0
1144 # determine number so that 0.2 <= total time < 2.0
1146 for index in range(0, 10):
1145 for index in range(0, 10):
1147 number = 10 ** index
1146 number = 10 ** index
1148 time_number = timer.timeit(number)
1147 time_number = timer.timeit(number)
1149 if time_number >= 0.2:
1148 if time_number >= 0.2:
1150 break
1149 break
1151
1150
1152 all_runs = timer.repeat(repeat, number)
1151 all_runs = timer.repeat(repeat, number)
1153 best = min(all_runs) / number
1152 best = min(all_runs) / number
1154 worst = max(all_runs) / number
1153 worst = max(all_runs) / number
1155 timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
1154 timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
1156
1155
1157 # Restore global vars from conflict_globs
1156 # Restore global vars from conflict_globs
1158 if local_ns is not None:
1157 if local_ns is not None:
1159 if len(conflict_globs) > 0:
1158 if len(conflict_globs) > 0:
1160 glob.update(conflict_globs)
1159 glob.update(conflict_globs)
1161
1160
1162 if not quiet :
1161 if not quiet :
1163 # Check best timing is greater than zero to avoid a
1162 # Check best timing is greater than zero to avoid a
1164 # ZeroDivisionError.
1163 # ZeroDivisionError.
1165 # In cases where the slowest timing is lesser than a micosecond
1164 # In cases where the slowest timing is lesser than a micosecond
1166 # we assume that it does not really matter if the fastest
1165 # we assume that it does not really matter if the fastest
1167 # timing is 4 times faster than the slowest timing or not.
1166 # timing is 4 times faster than the slowest timing or not.
1168 if worst > 4 * best and best > 0 and worst > 1e-6:
1167 if worst > 4 * best and best > 0 and worst > 1e-6:
1169 print("The slowest run took %0.2f times longer than the "
1168 print("The slowest run took %0.2f times longer than the "
1170 "fastest. This could mean that an intermediate result "
1169 "fastest. This could mean that an intermediate result "
1171 "is being cached." % (worst / best))
1170 "is being cached." % (worst / best))
1172
1171
1173 print( timeit_result )
1172 print( timeit_result )
1174
1173
1175 if tc > tc_min:
1174 if tc > tc_min:
1176 print("Compiler time: %.2f s" % tc)
1175 print("Compiler time: %.2f s" % tc)
1177 if return_result:
1176 if return_result:
1178 return timeit_result
1177 return timeit_result
1179
1178
1180 @skip_doctest
1179 @skip_doctest
1181 @no_var_expand
1180 @no_var_expand
1182 @needs_local_scope
1181 @needs_local_scope
1183 @line_cell_magic
1182 @line_cell_magic
1184 def time(self,line='', cell=None, local_ns=None):
1183 def time(self,line='', cell=None, local_ns=None):
1185 """Time execution of a Python statement or expression.
1184 """Time execution of a Python statement or expression.
1186
1185
1187 The CPU and wall clock times are printed, and the value of the
1186 The CPU and wall clock times are printed, and the value of the
1188 expression (if any) is returned. Note that under Win32, system time
1187 expression (if any) is returned. Note that under Win32, system time
1189 is always reported as 0, since it can not be measured.
1188 is always reported as 0, since it can not be measured.
1190
1189
1191 This function can be used both as a line and cell magic:
1190 This function can be used both as a line and cell magic:
1192
1191
1193 - In line mode you can time a single-line statement (though multiple
1192 - In line mode you can time a single-line statement (though multiple
1194 ones can be chained with using semicolons).
1193 ones can be chained with using semicolons).
1195
1194
1196 - In cell mode, you can time the cell body (a directly
1195 - In cell mode, you can time the cell body (a directly
1197 following statement raises an error).
1196 following statement raises an error).
1198
1197
1199 This function provides very basic timing functionality. Use the timeit
1198 This function provides very basic timing functionality. Use the timeit
1200 magic for more control over the measurement.
1199 magic for more control over the measurement.
1201
1200
1202 .. versionchanged:: 7.3
1201 .. versionchanged:: 7.3
1203 User variables are no longer expanded,
1202 User variables are no longer expanded,
1204 the magic line is always left unmodified.
1203 the magic line is always left unmodified.
1205
1204
1206 Examples
1205 Examples
1207 --------
1206 --------
1208 ::
1207 ::
1209
1208
1210 In [1]: %time 2**128
1209 In [1]: %time 2**128
1211 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1210 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1212 Wall time: 0.00
1211 Wall time: 0.00
1213 Out[1]: 340282366920938463463374607431768211456L
1212 Out[1]: 340282366920938463463374607431768211456L
1214
1213
1215 In [2]: n = 1000000
1214 In [2]: n = 1000000
1216
1215
1217 In [3]: %time sum(range(n))
1216 In [3]: %time sum(range(n))
1218 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1217 CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
1219 Wall time: 1.37
1218 Wall time: 1.37
1220 Out[3]: 499999500000L
1219 Out[3]: 499999500000L
1221
1220
1222 In [4]: %time print 'hello world'
1221 In [4]: %time print 'hello world'
1223 hello world
1222 hello world
1224 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1223 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1225 Wall time: 0.00
1224 Wall time: 0.00
1226
1225
1227 Note that the time needed by Python to compile the given expression
1226 Note that the time needed by Python to compile the given expression
1228 will be reported if it is more than 0.1s. In this example, the
1227 will be reported if it is more than 0.1s. In this example, the
1229 actual exponentiation is done by Python at compilation time, so while
1228 actual exponentiation is done by Python at compilation time, so while
1230 the expression can take a noticeable amount of time to compute, that
1229 the expression can take a noticeable amount of time to compute, that
1231 time is purely due to the compilation:
1230 time is purely due to the compilation:
1232
1231
1233 In [5]: %time 3**9999;
1232 In [5]: %time 3**9999;
1234 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1233 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1235 Wall time: 0.00 s
1234 Wall time: 0.00 s
1236
1235
1237 In [6]: %time 3**999999;
1236 In [6]: %time 3**999999;
1238 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1237 CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
1239 Wall time: 0.00 s
1238 Wall time: 0.00 s
1240 Compiler : 0.78 s
1239 Compiler : 0.78 s
1241 """
1240 """
1242
1241
1243 # fail immediately if the given expression can't be compiled
1242 # fail immediately if the given expression can't be compiled
1244
1243
1245 if line and cell:
1244 if line and cell:
1246 raise UsageError("Can't use statement directly after '%%time'!")
1245 raise UsageError("Can't use statement directly after '%%time'!")
1247
1246
1248 if cell:
1247 if cell:
1249 expr = self.shell.transform_cell(cell)
1248 expr = self.shell.transform_cell(cell)
1250 else:
1249 else:
1251 expr = self.shell.transform_cell(line)
1250 expr = self.shell.transform_cell(line)
1252
1251
1253 # Minimum time above which parse time will be reported
1252 # Minimum time above which parse time will be reported
1254 tp_min = 0.1
1253 tp_min = 0.1
1255
1254
1256 t0 = clock()
1255 t0 = clock()
1257 expr_ast = self.shell.compile.ast_parse(expr)
1256 expr_ast = self.shell.compile.ast_parse(expr)
1258 tp = clock()-t0
1257 tp = clock()-t0
1259
1258
1260 # Apply AST transformations
1259 # Apply AST transformations
1261 expr_ast = self.shell.transform_ast(expr_ast)
1260 expr_ast = self.shell.transform_ast(expr_ast)
1262
1261
1263 # Minimum time above which compilation time will be reported
1262 # Minimum time above which compilation time will be reported
1264 tc_min = 0.1
1263 tc_min = 0.1
1265
1264
1266 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1265 if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
1267 mode = 'eval'
1266 mode = 'eval'
1268 source = '<timed eval>'
1267 source = '<timed eval>'
1269 expr_ast = ast.Expression(expr_ast.body[0].value)
1268 expr_ast = ast.Expression(expr_ast.body[0].value)
1270 else:
1269 else:
1271 mode = 'exec'
1270 mode = 'exec'
1272 source = '<timed exec>'
1271 source = '<timed exec>'
1273 t0 = clock()
1272 t0 = clock()
1274 code = self.shell.compile(expr_ast, source, mode)
1273 code = self.shell.compile(expr_ast, source, mode)
1275 tc = clock()-t0
1274 tc = clock()-t0
1276
1275
1277 # skew measurement as little as possible
1276 # skew measurement as little as possible
1278 glob = self.shell.user_ns
1277 glob = self.shell.user_ns
1279 wtime = time.time
1278 wtime = time.time
1280 # time execution
1279 # time execution
1281 wall_st = wtime()
1280 wall_st = wtime()
1282 if mode=='eval':
1281 if mode=='eval':
1283 st = clock2()
1282 st = clock2()
1284 try:
1283 try:
1285 out = eval(code, glob, local_ns)
1284 out = eval(code, glob, local_ns)
1286 except:
1285 except:
1287 self.shell.showtraceback()
1286 self.shell.showtraceback()
1288 return
1287 return
1289 end = clock2()
1288 end = clock2()
1290 else:
1289 else:
1291 st = clock2()
1290 st = clock2()
1292 try:
1291 try:
1293 exec(code, glob, local_ns)
1292 exec(code, glob, local_ns)
1294 except:
1293 except:
1295 self.shell.showtraceback()
1294 self.shell.showtraceback()
1296 return
1295 return
1297 end = clock2()
1296 end = clock2()
1298 out = None
1297 out = None
1299 wall_end = wtime()
1298 wall_end = wtime()
1300 # Compute actual times and report
1299 # Compute actual times and report
1301 wall_time = wall_end-wall_st
1300 wall_time = wall_end-wall_st
1302 cpu_user = end[0]-st[0]
1301 cpu_user = end[0]-st[0]
1303 cpu_sys = end[1]-st[1]
1302 cpu_sys = end[1]-st[1]
1304 cpu_tot = cpu_user+cpu_sys
1303 cpu_tot = cpu_user+cpu_sys
1305 # On windows cpu_sys is always zero, so no new information to the next print
1304 # On windows cpu_sys is always zero, so no new information to the next print
1306 if sys.platform != 'win32':
1305 if sys.platform != 'win32':
1307 print("CPU times: user %s, sys: %s, total: %s" % \
1306 print("CPU times: user %s, sys: %s, total: %s" % \
1308 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
1307 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
1309 print("Wall time: %s" % _format_time(wall_time))
1308 print("Wall time: %s" % _format_time(wall_time))
1310 if tc > tc_min:
1309 if tc > tc_min:
1311 print("Compiler : %s" % _format_time(tc))
1310 print("Compiler : %s" % _format_time(tc))
1312 if tp > tp_min:
1311 if tp > tp_min:
1313 print("Parser : %s" % _format_time(tp))
1312 print("Parser : %s" % _format_time(tp))
1314 return out
1313 return out
1315
1314
1316 @skip_doctest
1315 @skip_doctest
1317 @line_magic
1316 @line_magic
1318 def macro(self, parameter_s=''):
1317 def macro(self, parameter_s=''):
1319 """Define a macro for future re-execution. It accepts ranges of history,
1318 """Define a macro for future re-execution. It accepts ranges of history,
1320 filenames or string objects.
1319 filenames or string objects.
1321
1320
1322 Usage:\\
1321 Usage:\\
1323 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1322 %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
1324
1323
1325 Options:
1324 Options:
1326
1325
1327 -r: use 'raw' input. By default, the 'processed' history is used,
1326 -r: use 'raw' input. By default, the 'processed' history is used,
1328 so that magics are loaded in their transformed version to valid
1327 so that magics are loaded in their transformed version to valid
1329 Python. If this option is given, the raw input as typed at the
1328 Python. If this option is given, the raw input as typed at the
1330 command line is used instead.
1329 command line is used instead.
1331
1330
1332 -q: quiet macro definition. By default, a tag line is printed
1331 -q: quiet macro definition. By default, a tag line is printed
1333 to indicate the macro has been created, and then the contents of
1332 to indicate the macro has been created, and then the contents of
1334 the macro are printed. If this option is given, then no printout
1333 the macro are printed. If this option is given, then no printout
1335 is produced once the macro is created.
1334 is produced once the macro is created.
1336
1335
1337 This will define a global variable called `name` which is a string
1336 This will define a global variable called `name` which is a string
1338 made of joining the slices and lines you specify (n1,n2,... numbers
1337 made of joining the slices and lines you specify (n1,n2,... numbers
1339 above) from your input history into a single string. This variable
1338 above) from your input history into a single string. This variable
1340 acts like an automatic function which re-executes those lines as if
1339 acts like an automatic function which re-executes those lines as if
1341 you had typed them. You just type 'name' at the prompt and the code
1340 you had typed them. You just type 'name' at the prompt and the code
1342 executes.
1341 executes.
1343
1342
1344 The syntax for indicating input ranges is described in %history.
1343 The syntax for indicating input ranges is described in %history.
1345
1344
1346 Note: as a 'hidden' feature, you can also use traditional python slice
1345 Note: as a 'hidden' feature, you can also use traditional python slice
1347 notation, where N:M means numbers N through M-1.
1346 notation, where N:M means numbers N through M-1.
1348
1347
1349 For example, if your history contains (print using %hist -n )::
1348 For example, if your history contains (print using %hist -n )::
1350
1349
1351 44: x=1
1350 44: x=1
1352 45: y=3
1351 45: y=3
1353 46: z=x+y
1352 46: z=x+y
1354 47: print x
1353 47: print x
1355 48: a=5
1354 48: a=5
1356 49: print 'x',x,'y',y
1355 49: print 'x',x,'y',y
1357
1356
1358 you can create a macro with lines 44 through 47 (included) and line 49
1357 you can create a macro with lines 44 through 47 (included) and line 49
1359 called my_macro with::
1358 called my_macro with::
1360
1359
1361 In [55]: %macro my_macro 44-47 49
1360 In [55]: %macro my_macro 44-47 49
1362
1361
1363 Now, typing `my_macro` (without quotes) will re-execute all this code
1362 Now, typing `my_macro` (without quotes) will re-execute all this code
1364 in one pass.
1363 in one pass.
1365
1364
1366 You don't need to give the line-numbers in order, and any given line
1365 You don't need to give the line-numbers in order, and any given line
1367 number can appear multiple times. You can assemble macros with any
1366 number can appear multiple times. You can assemble macros with any
1368 lines from your input history in any order.
1367 lines from your input history in any order.
1369
1368
1370 The macro is a simple object which holds its value in an attribute,
1369 The macro is a simple object which holds its value in an attribute,
1371 but IPython's display system checks for macros and executes them as
1370 but IPython's display system checks for macros and executes them as
1372 code instead of printing them when you type their name.
1371 code instead of printing them when you type their name.
1373
1372
1374 You can view a macro's contents by explicitly printing it with::
1373 You can view a macro's contents by explicitly printing it with::
1375
1374
1376 print macro_name
1375 print macro_name
1377
1376
1378 """
1377 """
1379 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1378 opts,args = self.parse_options(parameter_s,'rq',mode='list')
1380 if not args: # List existing macros
1379 if not args: # List existing macros
1381 return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
1380 return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
1382 if len(args) == 1:
1381 if len(args) == 1:
1383 raise UsageError(
1382 raise UsageError(
1384 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1383 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
1385 name, codefrom = args[0], " ".join(args[1:])
1384 name, codefrom = args[0], " ".join(args[1:])
1386
1385
1387 #print 'rng',ranges # dbg
1386 #print 'rng',ranges # dbg
1388 try:
1387 try:
1389 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1388 lines = self.shell.find_user_code(codefrom, 'r' in opts)
1390 except (ValueError, TypeError) as e:
1389 except (ValueError, TypeError) as e:
1391 print(e.args[0])
1390 print(e.args[0])
1392 return
1391 return
1393 macro = Macro(lines)
1392 macro = Macro(lines)
1394 self.shell.define_macro(name, macro)
1393 self.shell.define_macro(name, macro)
1395 if not ( 'q' in opts) :
1394 if not ( 'q' in opts) :
1396 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1395 print('Macro `%s` created. To execute, type its name (without quotes).' % name)
1397 print('=== Macro contents: ===')
1396 print('=== Macro contents: ===')
1398 print(macro, end=' ')
1397 print(macro, end=' ')
1399
1398
1400 @magic_arguments.magic_arguments()
1399 @magic_arguments.magic_arguments()
1401 @magic_arguments.argument('output', type=str, default='', nargs='?',
1400 @magic_arguments.argument('output', type=str, default='', nargs='?',
1402 help="""The name of the variable in which to store output.
1401 help="""The name of the variable in which to store output.
1403 This is a utils.io.CapturedIO object with stdout/err attributes
1402 This is a utils.io.CapturedIO object with stdout/err attributes
1404 for the text of the captured output.
1403 for the text of the captured output.
1405
1404
1406 CapturedOutput also has a show() method for displaying the output,
1405 CapturedOutput also has a show() method for displaying the output,
1407 and __call__ as well, so you can use that to quickly display the
1406 and __call__ as well, so you can use that to quickly display the
1408 output.
1407 output.
1409
1408
1410 If unspecified, captured output is discarded.
1409 If unspecified, captured output is discarded.
1411 """
1410 """
1412 )
1411 )
1413 @magic_arguments.argument('--no-stderr', action="store_true",
1412 @magic_arguments.argument('--no-stderr', action="store_true",
1414 help="""Don't capture stderr."""
1413 help="""Don't capture stderr."""
1415 )
1414 )
1416 @magic_arguments.argument('--no-stdout', action="store_true",
1415 @magic_arguments.argument('--no-stdout', action="store_true",
1417 help="""Don't capture stdout."""
1416 help="""Don't capture stdout."""
1418 )
1417 )
1419 @magic_arguments.argument('--no-display', action="store_true",
1418 @magic_arguments.argument('--no-display', action="store_true",
1420 help="""Don't capture IPython's rich display."""
1419 help="""Don't capture IPython's rich display."""
1421 )
1420 )
1422 @cell_magic
1421 @cell_magic
1423 def capture(self, line, cell):
1422 def capture(self, line, cell):
1424 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1423 """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
1425 args = magic_arguments.parse_argstring(self.capture, line)
1424 args = magic_arguments.parse_argstring(self.capture, line)
1426 out = not args.no_stdout
1425 out = not args.no_stdout
1427 err = not args.no_stderr
1426 err = not args.no_stderr
1428 disp = not args.no_display
1427 disp = not args.no_display
1429 with capture_output(out, err, disp) as io:
1428 with capture_output(out, err, disp) as io:
1430 self.shell.run_cell(cell)
1429 self.shell.run_cell(cell)
1431 if args.output:
1430 if args.output:
1432 self.shell.user_ns[args.output] = io
1431 self.shell.user_ns[args.output] = io
1433
1432
1434 def parse_breakpoint(text, current_file):
1433 def parse_breakpoint(text, current_file):
1435 '''Returns (file, line) for file:line and (current_file, line) for line'''
1434 '''Returns (file, line) for file:line and (current_file, line) for line'''
1436 colon = text.find(':')
1435 colon = text.find(':')
1437 if colon == -1:
1436 if colon == -1:
1438 return current_file, int(text)
1437 return current_file, int(text)
1439 else:
1438 else:
1440 return text[:colon], int(text[colon+1:])
1439 return text[:colon], int(text[colon+1:])
1441
1440
1442 def _format_time(timespan, precision=3):
1441 def _format_time(timespan, precision=3):
1443 """Formats the timespan in a human readable form"""
1442 """Formats the timespan in a human readable form"""
1444
1443
1445 if timespan >= 60.0:
1444 if timespan >= 60.0:
1446 # we have more than a minute, format that in a human readable form
1445 # we have more than a minute, format that in a human readable form
1447 # Idea from http://snipplr.com/view/5713/
1446 # Idea from http://snipplr.com/view/5713/
1448 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1447 parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
1449 time = []
1448 time = []
1450 leftover = timespan
1449 leftover = timespan
1451 for suffix, length in parts:
1450 for suffix, length in parts:
1452 value = int(leftover / length)
1451 value = int(leftover / length)
1453 if value > 0:
1452 if value > 0:
1454 leftover = leftover % length
1453 leftover = leftover % length
1455 time.append(u'%s%s' % (str(value), suffix))
1454 time.append(u'%s%s' % (str(value), suffix))
1456 if leftover < 1:
1455 if leftover < 1:
1457 break
1456 break
1458 return " ".join(time)
1457 return " ".join(time)
1459
1458
1460
1459
1461 # Unfortunately the unicode 'micro' symbol can cause problems in
1460 # Unfortunately the unicode 'micro' symbol can cause problems in
1462 # certain terminals.
1461 # certain terminals.
1463 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1462 # See bug: https://bugs.launchpad.net/ipython/+bug/348466
1464 # Try to prevent crashes by being more secure than it needs to
1463 # Try to prevent crashes by being more secure than it needs to
1465 # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
1464 # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
1466 units = [u"s", u"ms",u'us',"ns"] # the save value
1465 units = [u"s", u"ms",u'us',"ns"] # the save value
1467 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1466 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
1468 try:
1467 try:
1469 u'\xb5'.encode(sys.stdout.encoding)
1468 u'\xb5'.encode(sys.stdout.encoding)
1470 units = [u"s", u"ms",u'\xb5s',"ns"]
1469 units = [u"s", u"ms",u'\xb5s',"ns"]
1471 except:
1470 except:
1472 pass
1471 pass
1473 scaling = [1, 1e3, 1e6, 1e9]
1472 scaling = [1, 1e3, 1e6, 1e9]
1474
1473
1475 if timespan > 0.0:
1474 if timespan > 0.0:
1476 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1475 order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
1477 else:
1476 else:
1478 order = 3
1477 order = 3
1479 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
1478 return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
@@ -1,103 +1,104 b''
1 """Implementation of packaging-related magic functions.
1 """Implementation of packaging-related magic functions.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2018 The IPython Development Team.
4 # Copyright (c) 2018 The IPython Development Team.
5 #
5 #
6 # Distributed under the terms of the Modified BSD License.
6 # Distributed under the terms of the Modified BSD License.
7 #
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 import os
11 import os
12 import re
12 import re
13 import shlex
13 import shlex
14 import sys
14 import sys
15 from subprocess import Popen, PIPE
15 from subprocess import Popen, PIPE
16
16
17 from IPython.core.magic import Magics, magics_class, line_magic
17 from IPython.core.magic import Magics, magics_class, line_magic
18
18
19
19
20 def _is_conda_environment():
20 def _is_conda_environment():
21 """Return True if the current Python executable is in a conda env"""
21 """Return True if the current Python executable is in a conda env"""
22 # TODO: does this need to change on windows?
22 # TODO: does this need to change on windows?
23 conda_history = os.path.join(sys.prefix, 'conda-meta', 'history')
23 conda_history = os.path.join(sys.prefix, 'conda-meta', 'history')
24 return os.path.exists(conda_history)
24 return os.path.exists(conda_history)
25
25
26
26
27 def _get_conda_executable():
27 def _get_conda_executable():
28 """Find the path to the conda executable"""
28 """Find the path to the conda executable"""
29 # Check if there is a conda executable in the same directory as the Python executable.
29 # Check if there is a conda executable in the same directory as the Python executable.
30 # This is the case within conda's root environment.
30 # This is the case within conda's root environment.
31 conda = os.path.join(os.path.dirname(sys.executable), 'conda')
31 conda = os.path.join(os.path.dirname(sys.executable), 'conda')
32 if os.path.isfile(conda):
32 if os.path.isfile(conda):
33 return conda
33 return conda
34
34
35 # Otherwise, attempt to extract the executable from conda history.
35 # Otherwise, attempt to extract the executable from conda history.
36 # This applies in any conda environment.
36 # This applies in any conda environment.
37 R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]")
37 R = re.compile(r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]")
38 for line in open(os.path.join(sys.prefix, 'conda-meta', 'history')):
38 with open(os.path.join(sys.prefix, 'conda-meta', 'history')) as f:
39 for line in f:
39 match = R.match(line)
40 match = R.match(line)
40 if match:
41 if match:
41 return match.groupdict()['command']
42 return match.groupdict()['command']
42
43
43 # Fallback: assume conda is available on the system path.
44 # Fallback: assume conda is available on the system path.
44 return "conda"
45 return "conda"
45
46
46
47
47 CONDA_COMMANDS_REQUIRING_PREFIX = {
48 CONDA_COMMANDS_REQUIRING_PREFIX = {
48 'install', 'list', 'remove', 'uninstall', 'update', 'upgrade',
49 'install', 'list', 'remove', 'uninstall', 'update', 'upgrade',
49 }
50 }
50 CONDA_COMMANDS_REQUIRING_YES = {
51 CONDA_COMMANDS_REQUIRING_YES = {
51 'install', 'remove', 'uninstall', 'update', 'upgrade',
52 'install', 'remove', 'uninstall', 'update', 'upgrade',
52 }
53 }
53 CONDA_ENV_FLAGS = {'-p', '--prefix', '-n', '--name'}
54 CONDA_ENV_FLAGS = {'-p', '--prefix', '-n', '--name'}
54 CONDA_YES_FLAGS = {'-y', '--y'}
55 CONDA_YES_FLAGS = {'-y', '--y'}
55
56
56
57
57 @magics_class
58 @magics_class
58 class PackagingMagics(Magics):
59 class PackagingMagics(Magics):
59 """Magics related to packaging & installation"""
60 """Magics related to packaging & installation"""
60
61
61 @line_magic
62 @line_magic
62 def pip(self, line):
63 def pip(self, line):
63 """Run the pip package manager within the current kernel.
64 """Run the pip package manager within the current kernel.
64
65
65 Usage:
66 Usage:
66 %pip install [pkgs]
67 %pip install [pkgs]
67 """
68 """
68 self.shell.system(' '.join([sys.executable, '-m', 'pip', line]))
69 self.shell.system(' '.join([sys.executable, '-m', 'pip', line]))
69 print("Note: you may need to restart the kernel to use updated packages.")
70 print("Note: you may need to restart the kernel to use updated packages.")
70
71
71 @line_magic
72 @line_magic
72 def conda(self, line):
73 def conda(self, line):
73 """Run the conda package manager within the current kernel.
74 """Run the conda package manager within the current kernel.
74
75
75 Usage:
76 Usage:
76 %conda install [pkgs]
77 %conda install [pkgs]
77 """
78 """
78 if not _is_conda_environment():
79 if not _is_conda_environment():
79 raise ValueError("The python kernel does not appear to be a conda environment. "
80 raise ValueError("The python kernel does not appear to be a conda environment. "
80 "Please use ``%pip install`` instead.")
81 "Please use ``%pip install`` instead.")
81
82
82 conda = _get_conda_executable()
83 conda = _get_conda_executable()
83 args = shlex.split(line)
84 args = shlex.split(line)
84 command = args[0]
85 command = args[0]
85 args = args[1:]
86 args = args[1:]
86 extra_args = []
87 extra_args = []
87
88
88 # When the subprocess does not allow us to respond "yes" during the installation,
89 # When the subprocess does not allow us to respond "yes" during the installation,
89 # we need to insert --yes in the argument list for some commands
90 # we need to insert --yes in the argument list for some commands
90 stdin_disabled = getattr(self.shell, 'kernel', None) is not None
91 stdin_disabled = getattr(self.shell, 'kernel', None) is not None
91 needs_yes = command in CONDA_COMMANDS_REQUIRING_YES
92 needs_yes = command in CONDA_COMMANDS_REQUIRING_YES
92 has_yes = set(args).intersection(CONDA_YES_FLAGS)
93 has_yes = set(args).intersection(CONDA_YES_FLAGS)
93 if stdin_disabled and needs_yes and not has_yes:
94 if stdin_disabled and needs_yes and not has_yes:
94 extra_args.append("--yes")
95 extra_args.append("--yes")
95
96
96 # Add --prefix to point conda installation to the current environment
97 # Add --prefix to point conda installation to the current environment
97 needs_prefix = command in CONDA_COMMANDS_REQUIRING_PREFIX
98 needs_prefix = command in CONDA_COMMANDS_REQUIRING_PREFIX
98 has_prefix = set(args).intersection(CONDA_ENV_FLAGS)
99 has_prefix = set(args).intersection(CONDA_ENV_FLAGS)
99 if needs_prefix and not has_prefix:
100 if needs_prefix and not has_prefix:
100 extra_args.extend(["--prefix", sys.prefix])
101 extra_args.extend(["--prefix", sys.prefix])
101
102
102 self.shell.system(' '.join([conda, command] + extra_args + args))
103 self.shell.system(' '.join([conda, command] + extra_args + args))
103 print("\nNote: you may need to restart the kernel to use updated packages.") No newline at end of file
104 print("\nNote: you may need to restart the kernel to use updated packages.")
@@ -1,226 +1,225 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 %store magic for lightweight persistence.
3 %store magic for lightweight persistence.
4
4
5 Stores variables, aliases and macros in IPython's database.
5 Stores variables, aliases and macros in IPython's database.
6
6
7 To automatically restore stored variables at startup, add this to your
7 To automatically restore stored variables at startup, add this to your
8 :file:`ipython_config.py` file::
8 :file:`ipython_config.py` file::
9
9
10 c.StoreMagics.autorestore = True
10 c.StoreMagics.autorestore = True
11 """
11 """
12
12
13 # Copyright (c) IPython Development Team.
13 # Copyright (c) IPython Development Team.
14 # Distributed under the terms of the Modified BSD License.
14 # Distributed under the terms of the Modified BSD License.
15
15
16 import inspect, os, sys, textwrap
16 import inspect, os, sys, textwrap
17
17
18 from IPython.core.error import UsageError
18 from IPython.core.error import UsageError
19 from IPython.core.magic import Magics, magics_class, line_magic
19 from IPython.core.magic import Magics, magics_class, line_magic
20 from traitlets import Bool
20 from traitlets import Bool
21
21
22
22
23 def restore_aliases(ip):
23 def restore_aliases(ip):
24 staliases = ip.db.get('stored_aliases', {})
24 staliases = ip.db.get('stored_aliases', {})
25 for k,v in staliases.items():
25 for k,v in staliases.items():
26 #print "restore alias",k,v # dbg
26 #print "restore alias",k,v # dbg
27 #self.alias_table[k] = v
27 #self.alias_table[k] = v
28 ip.alias_manager.define_alias(k,v)
28 ip.alias_manager.define_alias(k,v)
29
29
30
30
31 def refresh_variables(ip):
31 def refresh_variables(ip):
32 db = ip.db
32 db = ip.db
33 for key in db.keys('autorestore/*'):
33 for key in db.keys('autorestore/*'):
34 # strip autorestore
34 # strip autorestore
35 justkey = os.path.basename(key)
35 justkey = os.path.basename(key)
36 try:
36 try:
37 obj = db[key]
37 obj = db[key]
38 except KeyError:
38 except KeyError:
39 print("Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey)
39 print("Unable to restore variable '%s', ignoring (use %%store -d to forget!)" % justkey)
40 print("The error was:", sys.exc_info()[0])
40 print("The error was:", sys.exc_info()[0])
41 else:
41 else:
42 #print "restored",justkey,"=",obj #dbg
42 #print "restored",justkey,"=",obj #dbg
43 ip.user_ns[justkey] = obj
43 ip.user_ns[justkey] = obj
44
44
45
45
46 def restore_dhist(ip):
46 def restore_dhist(ip):
47 ip.user_ns['_dh'] = ip.db.get('dhist',[])
47 ip.user_ns['_dh'] = ip.db.get('dhist',[])
48
48
49
49
50 def restore_data(ip):
50 def restore_data(ip):
51 refresh_variables(ip)
51 refresh_variables(ip)
52 restore_aliases(ip)
52 restore_aliases(ip)
53 restore_dhist(ip)
53 restore_dhist(ip)
54
54
55
55
56 @magics_class
56 @magics_class
57 class StoreMagics(Magics):
57 class StoreMagics(Magics):
58 """Lightweight persistence for python variables.
58 """Lightweight persistence for python variables.
59
59
60 Provides the %store magic."""
60 Provides the %store magic."""
61
61
62 autorestore = Bool(False, help=
62 autorestore = Bool(False, help=
63 """If True, any %store-d variables will be automatically restored
63 """If True, any %store-d variables will be automatically restored
64 when IPython starts.
64 when IPython starts.
65 """
65 """
66 ).tag(config=True)
66 ).tag(config=True)
67
67
68 def __init__(self, shell):
68 def __init__(self, shell):
69 super(StoreMagics, self).__init__(shell=shell)
69 super(StoreMagics, self).__init__(shell=shell)
70 self.shell.configurables.append(self)
70 self.shell.configurables.append(self)
71 if self.autorestore:
71 if self.autorestore:
72 restore_data(self.shell)
72 restore_data(self.shell)
73
73
74 @line_magic
74 @line_magic
75 def store(self, parameter_s=''):
75 def store(self, parameter_s=''):
76 """Lightweight persistence for python variables.
76 """Lightweight persistence for python variables.
77
77
78 Example::
78 Example::
79
79
80 In [1]: l = ['hello',10,'world']
80 In [1]: l = ['hello',10,'world']
81 In [2]: %store l
81 In [2]: %store l
82 In [3]: exit
82 In [3]: exit
83
83
84 (IPython session is closed and started again...)
84 (IPython session is closed and started again...)
85
85
86 ville@badger:~$ ipython
86 ville@badger:~$ ipython
87 In [1]: l
87 In [1]: l
88 NameError: name 'l' is not defined
88 NameError: name 'l' is not defined
89 In [2]: %store -r
89 In [2]: %store -r
90 In [3]: l
90 In [3]: l
91 Out[3]: ['hello', 10, 'world']
91 Out[3]: ['hello', 10, 'world']
92
92
93 Usage:
93 Usage:
94
94
95 * ``%store`` - Show list of all variables and their current
95 * ``%store`` - Show list of all variables and their current
96 values
96 values
97 * ``%store spam`` - Store the *current* value of the variable spam
97 * ``%store spam`` - Store the *current* value of the variable spam
98 to disk
98 to disk
99 * ``%store -d spam`` - Remove the variable and its value from storage
99 * ``%store -d spam`` - Remove the variable and its value from storage
100 * ``%store -z`` - Remove all variables from storage
100 * ``%store -z`` - Remove all variables from storage
101 * ``%store -r`` - Refresh all variables from store (overwrite
101 * ``%store -r`` - Refresh all variables from store (overwrite
102 current vals)
102 current vals)
103 * ``%store -r spam bar`` - Refresh specified variables from store
103 * ``%store -r spam bar`` - Refresh specified variables from store
104 (delete current val)
104 (delete current val)
105 * ``%store foo >a.txt`` - Store value of foo to new file a.txt
105 * ``%store foo >a.txt`` - Store value of foo to new file a.txt
106 * ``%store foo >>a.txt`` - Append value of foo to file a.txt
106 * ``%store foo >>a.txt`` - Append value of foo to file a.txt
107
107
108 It should be noted that if you change the value of a variable, you
108 It should be noted that if you change the value of a variable, you
109 need to %store it again if you want to persist the new value.
109 need to %store it again if you want to persist the new value.
110
110
111 Note also that the variables will need to be pickleable; most basic
111 Note also that the variables will need to be pickleable; most basic
112 python types can be safely %store'd.
112 python types can be safely %store'd.
113
113
114 Also aliases can be %store'd across sessions.
114 Also aliases can be %store'd across sessions.
115 """
115 """
116
116
117 opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
117 opts,argsl = self.parse_options(parameter_s,'drz',mode='string')
118 args = argsl.split(None,1)
118 args = argsl.split(None,1)
119 ip = self.shell
119 ip = self.shell
120 db = ip.db
120 db = ip.db
121 # delete
121 # delete
122 if 'd' in opts:
122 if 'd' in opts:
123 try:
123 try:
124 todel = args[0]
124 todel = args[0]
125 except IndexError:
125 except IndexError:
126 raise UsageError('You must provide the variable to forget')
126 raise UsageError('You must provide the variable to forget')
127 else:
127 else:
128 try:
128 try:
129 del db['autorestore/' + todel]
129 del db['autorestore/' + todel]
130 except:
130 except:
131 raise UsageError("Can't delete variable '%s'" % todel)
131 raise UsageError("Can't delete variable '%s'" % todel)
132 # reset
132 # reset
133 elif 'z' in opts:
133 elif 'z' in opts:
134 for k in db.keys('autorestore/*'):
134 for k in db.keys('autorestore/*'):
135 del db[k]
135 del db[k]
136
136
137 elif 'r' in opts:
137 elif 'r' in opts:
138 if args:
138 if args:
139 for arg in args:
139 for arg in args:
140 try:
140 try:
141 obj = db['autorestore/' + arg]
141 obj = db['autorestore/' + arg]
142 except KeyError:
142 except KeyError:
143 print("no stored variable %s" % arg)
143 print("no stored variable %s" % arg)
144 else:
144 else:
145 ip.user_ns[arg] = obj
145 ip.user_ns[arg] = obj
146 else:
146 else:
147 restore_data(ip)
147 restore_data(ip)
148
148
149 # run without arguments -> list variables & values
149 # run without arguments -> list variables & values
150 elif not args:
150 elif not args:
151 vars = db.keys('autorestore/*')
151 vars = db.keys('autorestore/*')
152 vars.sort()
152 vars.sort()
153 if vars:
153 if vars:
154 size = max(map(len, vars))
154 size = max(map(len, vars))
155 else:
155 else:
156 size = 0
156 size = 0
157
157
158 print('Stored variables and their in-db values:')
158 print('Stored variables and their in-db values:')
159 fmt = '%-'+str(size)+'s -> %s'
159 fmt = '%-'+str(size)+'s -> %s'
160 get = db.get
160 get = db.get
161 for var in vars:
161 for var in vars:
162 justkey = os.path.basename(var)
162 justkey = os.path.basename(var)
163 # print 30 first characters from every var
163 # print 30 first characters from every var
164 print(fmt % (justkey, repr(get(var, '<unavailable>'))[:50]))
164 print(fmt % (justkey, repr(get(var, '<unavailable>'))[:50]))
165
165
166 # default action - store the variable
166 # default action - store the variable
167 else:
167 else:
168 # %store foo >file.txt or >>file.txt
168 # %store foo >file.txt or >>file.txt
169 if len(args) > 1 and args[1].startswith('>'):
169 if len(args) > 1 and args[1].startswith('>'):
170 fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
170 fnam = os.path.expanduser(args[1].lstrip('>').lstrip())
171 if args[1].startswith('>>'):
171 if args[1].startswith('>>'):
172 fil = open(fnam, 'a')
172 fil = open(fnam, 'a')
173 else:
173 else:
174 fil = open(fnam, 'w')
174 fil = open(fnam, 'w')
175 with fil:
175 obj = ip.ev(args[0])
176 obj = ip.ev(args[0])
176 print("Writing '%s' (%s) to file '%s'." % (args[0],
177 print("Writing '%s' (%s) to file '%s'." % (args[0],
177 obj.__class__.__name__, fnam))
178 obj.__class__.__name__, fnam))
178
179
179
180 if not isinstance (obj, str):
180 if not isinstance (obj, str):
181 from pprint import pprint
181 from pprint import pprint
182 pprint(obj, fil)
182 pprint(obj, fil)
183 else:
183 else:
184 fil.write(obj)
184 fil.write(obj)
185 if not obj.endswith('\n'):
185 if not obj.endswith('\n'):
186 fil.write('\n')
186 fil.write('\n')
187
187
188 fil.close()
189 return
188 return
190
189
191 # %store foo
190 # %store foo
192 try:
191 try:
193 obj = ip.user_ns[args[0]]
192 obj = ip.user_ns[args[0]]
194 except KeyError:
193 except KeyError:
195 # it might be an alias
194 # it might be an alias
196 name = args[0]
195 name = args[0]
197 try:
196 try:
198 cmd = ip.alias_manager.retrieve_alias(name)
197 cmd = ip.alias_manager.retrieve_alias(name)
199 except ValueError:
198 except ValueError:
200 raise UsageError("Unknown variable '%s'" % name)
199 raise UsageError("Unknown variable '%s'" % name)
201
200
202 staliases = db.get('stored_aliases',{})
201 staliases = db.get('stored_aliases',{})
203 staliases[name] = cmd
202 staliases[name] = cmd
204 db['stored_aliases'] = staliases
203 db['stored_aliases'] = staliases
205 print("Alias stored: %s (%s)" % (name, cmd))
204 print("Alias stored: %s (%s)" % (name, cmd))
206 return
205 return
207
206
208 else:
207 else:
209 modname = getattr(inspect.getmodule(obj), '__name__', '')
208 modname = getattr(inspect.getmodule(obj), '__name__', '')
210 if modname == '__main__':
209 if modname == '__main__':
211 print(textwrap.dedent("""\
210 print(textwrap.dedent("""\
212 Warning:%s is %s
211 Warning:%s is %s
213 Proper storage of interactively declared classes (or instances
212 Proper storage of interactively declared classes (or instances
214 of those classes) is not possible! Only instances
213 of those classes) is not possible! Only instances
215 of classes in real modules on file system can be %%store'd.
214 of classes in real modules on file system can be %%store'd.
216 """ % (args[0], obj) ))
215 """ % (args[0], obj) ))
217 return
216 return
218 #pickled = pickle.dumps(obj)
217 #pickled = pickle.dumps(obj)
219 db[ 'autorestore/' + args[0] ] = obj
218 db[ 'autorestore/' + args[0] ] = obj
220 print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__))
219 print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__))
221
220
222
221
223 def load_ipython_extension(ip):
222 def load_ipython_extension(ip):
224 """Load the extension in IPython."""
223 """Load the extension in IPython."""
225 ip.register_magics(StoreMagics)
224 ip.register_magics(StoreMagics)
226
225
@@ -1,404 +1,398 b''
1 """Tests for autoreload extension.
1 """Tests for autoreload extension.
2 """
2 """
3 #-----------------------------------------------------------------------------
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2012 IPython Development Team.
4 # Copyright (c) 2012 IPython Development Team.
5 #
5 #
6 # Distributed under the terms of the Modified BSD License.
6 # Distributed under the terms of the Modified BSD License.
7 #
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15 import os
15 import os
16 import sys
16 import sys
17 import tempfile
17 import tempfile
18 import textwrap
18 import textwrap
19 import shutil
19 import shutil
20 import random
20 import random
21 import time
21 import time
22 from io import StringIO
22 from io import StringIO
23
23
24 import nose.tools as nt
24 import nose.tools as nt
25 import IPython.testing.tools as tt
25 import IPython.testing.tools as tt
26
26
27 from IPython.testing.decorators import skipif
27 from IPython.testing.decorators import skipif
28
28
29 from IPython.extensions.autoreload import AutoreloadMagics
29 from IPython.extensions.autoreload import AutoreloadMagics
30 from IPython.core.events import EventManager, pre_run_cell
30 from IPython.core.events import EventManager, pre_run_cell
31
31
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33 # Test fixture
33 # Test fixture
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35
35
36 noop = lambda *a, **kw: None
36 noop = lambda *a, **kw: None
37
37
38 class FakeShell(object):
38 class FakeShell(object):
39
39
40 def __init__(self):
40 def __init__(self):
41 self.ns = {}
41 self.ns = {}
42 self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
42 self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
43 self.auto_magics = AutoreloadMagics(shell=self)
43 self.auto_magics = AutoreloadMagics(shell=self)
44 self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
44 self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
45
45
46 register_magics = set_hook = noop
46 register_magics = set_hook = noop
47
47
48 def run_code(self, code):
48 def run_code(self, code):
49 self.events.trigger('pre_run_cell')
49 self.events.trigger('pre_run_cell')
50 exec(code, self.ns)
50 exec(code, self.ns)
51 self.auto_magics.post_execute_hook()
51 self.auto_magics.post_execute_hook()
52
52
53 def push(self, items):
53 def push(self, items):
54 self.ns.update(items)
54 self.ns.update(items)
55
55
56 def magic_autoreload(self, parameter):
56 def magic_autoreload(self, parameter):
57 self.auto_magics.autoreload(parameter)
57 self.auto_magics.autoreload(parameter)
58
58
59 def magic_aimport(self, parameter, stream=None):
59 def magic_aimport(self, parameter, stream=None):
60 self.auto_magics.aimport(parameter, stream=stream)
60 self.auto_magics.aimport(parameter, stream=stream)
61 self.auto_magics.post_execute_hook()
61 self.auto_magics.post_execute_hook()
62
62
63
63
64 class Fixture(object):
64 class Fixture(object):
65 """Fixture for creating test module files"""
65 """Fixture for creating test module files"""
66
66
67 test_dir = None
67 test_dir = None
68 old_sys_path = None
68 old_sys_path = None
69 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
69 filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
70
70
71 def setUp(self):
71 def setUp(self):
72 self.test_dir = tempfile.mkdtemp()
72 self.test_dir = tempfile.mkdtemp()
73 self.old_sys_path = list(sys.path)
73 self.old_sys_path = list(sys.path)
74 sys.path.insert(0, self.test_dir)
74 sys.path.insert(0, self.test_dir)
75 self.shell = FakeShell()
75 self.shell = FakeShell()
76
76
77 def tearDown(self):
77 def tearDown(self):
78 shutil.rmtree(self.test_dir)
78 shutil.rmtree(self.test_dir)
79 sys.path = self.old_sys_path
79 sys.path = self.old_sys_path
80
80
81 self.test_dir = None
81 self.test_dir = None
82 self.old_sys_path = None
82 self.old_sys_path = None
83 self.shell = None
83 self.shell = None
84
84
85 def get_module(self):
85 def get_module(self):
86 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
86 module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
87 if module_name in sys.modules:
87 if module_name in sys.modules:
88 del sys.modules[module_name]
88 del sys.modules[module_name]
89 file_name = os.path.join(self.test_dir, module_name + ".py")
89 file_name = os.path.join(self.test_dir, module_name + ".py")
90 return module_name, file_name
90 return module_name, file_name
91
91
92 def write_file(self, filename, content):
92 def write_file(self, filename, content):
93 """
93 """
94 Write a file, and force a timestamp difference of at least one second
94 Write a file, and force a timestamp difference of at least one second
95
95
96 Notes
96 Notes
97 -----
97 -----
98 Python's .pyc files record the timestamp of their compilation
98 Python's .pyc files record the timestamp of their compilation
99 with a time resolution of one second.
99 with a time resolution of one second.
100
100
101 Therefore, we need to force a timestamp difference between .py
101 Therefore, we need to force a timestamp difference between .py
102 and .pyc, without having the .py file be timestamped in the
102 and .pyc, without having the .py file be timestamped in the
103 future, and without changing the timestamp of the .pyc file
103 future, and without changing the timestamp of the .pyc file
104 (because that is stored in the file). The only reliable way
104 (because that is stored in the file). The only reliable way
105 to achieve this seems to be to sleep.
105 to achieve this seems to be to sleep.
106 """
106 """
107
107
108 # Sleep one second + eps
108 # Sleep one second + eps
109 time.sleep(1.05)
109 time.sleep(1.05)
110
110
111 # Write
111 # Write
112 f = open(filename, 'w')
112 with open(filename, 'w') as f:
113 try:
114 f.write(content)
113 f.write(content)
115 finally:
116 f.close()
117
114
118 def new_module(self, code):
115 def new_module(self, code):
119 mod_name, mod_fn = self.get_module()
116 mod_name, mod_fn = self.get_module()
120 f = open(mod_fn, 'w')
117 with open(mod_fn, 'w') as f:
121 try:
122 f.write(code)
118 f.write(code)
123 finally:
124 f.close()
125 return mod_name, mod_fn
119 return mod_name, mod_fn
126
120
127 #-----------------------------------------------------------------------------
121 #-----------------------------------------------------------------------------
128 # Test automatic reloading
122 # Test automatic reloading
129 #-----------------------------------------------------------------------------
123 #-----------------------------------------------------------------------------
130
124
131 class TestAutoreload(Fixture):
125 class TestAutoreload(Fixture):
132
126
133 @skipif(sys.version_info < (3, 6))
127 @skipif(sys.version_info < (3, 6))
134 def test_reload_enums(self):
128 def test_reload_enums(self):
135 import enum
129 import enum
136 mod_name, mod_fn = self.new_module(textwrap.dedent("""
130 mod_name, mod_fn = self.new_module(textwrap.dedent("""
137 from enum import Enum
131 from enum import Enum
138 class MyEnum(Enum):
132 class MyEnum(Enum):
139 A = 'A'
133 A = 'A'
140 B = 'B'
134 B = 'B'
141 """))
135 """))
142 self.shell.magic_autoreload("2")
136 self.shell.magic_autoreload("2")
143 self.shell.magic_aimport(mod_name)
137 self.shell.magic_aimport(mod_name)
144 self.write_file(mod_fn, textwrap.dedent("""
138 self.write_file(mod_fn, textwrap.dedent("""
145 from enum import Enum
139 from enum import Enum
146 class MyEnum(Enum):
140 class MyEnum(Enum):
147 A = 'A'
141 A = 'A'
148 B = 'B'
142 B = 'B'
149 C = 'C'
143 C = 'C'
150 """))
144 """))
151 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
145 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
152 self.shell.run_code("pass") # trigger another reload
146 self.shell.run_code("pass") # trigger another reload
153
147
154 def test_reload_class_attributes(self):
148 def test_reload_class_attributes(self):
155 self.shell.magic_autoreload("2")
149 self.shell.magic_autoreload("2")
156 mod_name, mod_fn = self.new_module(textwrap.dedent("""
150 mod_name, mod_fn = self.new_module(textwrap.dedent("""
157 class MyClass:
151 class MyClass:
158
152
159 def __init__(self, a=10):
153 def __init__(self, a=10):
160 self.a = a
154 self.a = a
161 self.b = 22
155 self.b = 22
162 # self.toto = 33
156 # self.toto = 33
163
157
164 def square(self):
158 def square(self):
165 print('compute square')
159 print('compute square')
166 return self.a*self.a
160 return self.a*self.a
167 """
161 """
168 )
162 )
169 )
163 )
170 self.shell.run_code("from %s import MyClass" % mod_name)
164 self.shell.run_code("from %s import MyClass" % mod_name)
171 self.shell.run_code("first = MyClass(5)")
165 self.shell.run_code("first = MyClass(5)")
172 self.shell.run_code("first.square()")
166 self.shell.run_code("first.square()")
173 with nt.assert_raises(AttributeError):
167 with nt.assert_raises(AttributeError):
174 self.shell.run_code("first.cube()")
168 self.shell.run_code("first.cube()")
175 with nt.assert_raises(AttributeError):
169 with nt.assert_raises(AttributeError):
176 self.shell.run_code("first.power(5)")
170 self.shell.run_code("first.power(5)")
177 self.shell.run_code("first.b")
171 self.shell.run_code("first.b")
178 with nt.assert_raises(AttributeError):
172 with nt.assert_raises(AttributeError):
179 self.shell.run_code("first.toto")
173 self.shell.run_code("first.toto")
180
174
181 # remove square, add power
175 # remove square, add power
182
176
183 self.write_file(
177 self.write_file(
184 mod_fn,
178 mod_fn,
185 textwrap.dedent(
179 textwrap.dedent(
186 """
180 """
187 class MyClass:
181 class MyClass:
188
182
189 def __init__(self, a=10):
183 def __init__(self, a=10):
190 self.a = a
184 self.a = a
191 self.b = 11
185 self.b = 11
192
186
193 def power(self, p):
187 def power(self, p):
194 print('compute power '+str(p))
188 print('compute power '+str(p))
195 return self.a**p
189 return self.a**p
196 """
190 """
197 ),
191 ),
198 )
192 )
199
193
200 self.shell.run_code("second = MyClass(5)")
194 self.shell.run_code("second = MyClass(5)")
201
195
202 for object_name in {'first', 'second'}:
196 for object_name in {'first', 'second'}:
203 self.shell.run_code("{object_name}.power(5)".format(object_name=object_name))
197 self.shell.run_code("{object_name}.power(5)".format(object_name=object_name))
204 with nt.assert_raises(AttributeError):
198 with nt.assert_raises(AttributeError):
205 self.shell.run_code("{object_name}.cube()".format(object_name=object_name))
199 self.shell.run_code("{object_name}.cube()".format(object_name=object_name))
206 with nt.assert_raises(AttributeError):
200 with nt.assert_raises(AttributeError):
207 self.shell.run_code("{object_name}.square()".format(object_name=object_name))
201 self.shell.run_code("{object_name}.square()".format(object_name=object_name))
208 self.shell.run_code("{object_name}.b".format(object_name=object_name))
202 self.shell.run_code("{object_name}.b".format(object_name=object_name))
209 self.shell.run_code("{object_name}.a".format(object_name=object_name))
203 self.shell.run_code("{object_name}.a".format(object_name=object_name))
210 with nt.assert_raises(AttributeError):
204 with nt.assert_raises(AttributeError):
211 self.shell.run_code("{object_name}.toto".format(object_name=object_name))
205 self.shell.run_code("{object_name}.toto".format(object_name=object_name))
212
206
213 def _check_smoketest(self, use_aimport=True):
207 def _check_smoketest(self, use_aimport=True):
214 """
208 """
215 Functional test for the automatic reloader using either
209 Functional test for the automatic reloader using either
216 '%autoreload 1' or '%autoreload 2'
210 '%autoreload 1' or '%autoreload 2'
217 """
211 """
218
212
219 mod_name, mod_fn = self.new_module("""
213 mod_name, mod_fn = self.new_module("""
220 x = 9
214 x = 9
221
215
222 z = 123 # this item will be deleted
216 z = 123 # this item will be deleted
223
217
224 def foo(y):
218 def foo(y):
225 return y + 3
219 return y + 3
226
220
227 class Baz(object):
221 class Baz(object):
228 def __init__(self, x):
222 def __init__(self, x):
229 self.x = x
223 self.x = x
230 def bar(self, y):
224 def bar(self, y):
231 return self.x + y
225 return self.x + y
232 @property
226 @property
233 def quux(self):
227 def quux(self):
234 return 42
228 return 42
235 def zzz(self):
229 def zzz(self):
236 '''This method will be deleted below'''
230 '''This method will be deleted below'''
237 return 99
231 return 99
238
232
239 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
233 class Bar: # old-style class: weakref doesn't work for it on Python < 2.7
240 def foo(self):
234 def foo(self):
241 return 1
235 return 1
242 """)
236 """)
243
237
244 #
238 #
245 # Import module, and mark for reloading
239 # Import module, and mark for reloading
246 #
240 #
247 if use_aimport:
241 if use_aimport:
248 self.shell.magic_autoreload("1")
242 self.shell.magic_autoreload("1")
249 self.shell.magic_aimport(mod_name)
243 self.shell.magic_aimport(mod_name)
250 stream = StringIO()
244 stream = StringIO()
251 self.shell.magic_aimport("", stream=stream)
245 self.shell.magic_aimport("", stream=stream)
252 nt.assert_in(("Modules to reload:\n%s" % mod_name), stream.getvalue())
246 nt.assert_in(("Modules to reload:\n%s" % mod_name), stream.getvalue())
253
247
254 with nt.assert_raises(ImportError):
248 with nt.assert_raises(ImportError):
255 self.shell.magic_aimport("tmpmod_as318989e89ds")
249 self.shell.magic_aimport("tmpmod_as318989e89ds")
256 else:
250 else:
257 self.shell.magic_autoreload("2")
251 self.shell.magic_autoreload("2")
258 self.shell.run_code("import %s" % mod_name)
252 self.shell.run_code("import %s" % mod_name)
259 stream = StringIO()
253 stream = StringIO()
260 self.shell.magic_aimport("", stream=stream)
254 self.shell.magic_aimport("", stream=stream)
261 nt.assert_true("Modules to reload:\nall-except-skipped" in
255 nt.assert_true("Modules to reload:\nall-except-skipped" in
262 stream.getvalue())
256 stream.getvalue())
263 nt.assert_in(mod_name, self.shell.ns)
257 nt.assert_in(mod_name, self.shell.ns)
264
258
265 mod = sys.modules[mod_name]
259 mod = sys.modules[mod_name]
266
260
267 #
261 #
268 # Test module contents
262 # Test module contents
269 #
263 #
270 old_foo = mod.foo
264 old_foo = mod.foo
271 old_obj = mod.Baz(9)
265 old_obj = mod.Baz(9)
272 old_obj2 = mod.Bar()
266 old_obj2 = mod.Bar()
273
267
274 def check_module_contents():
268 def check_module_contents():
275 nt.assert_equal(mod.x, 9)
269 nt.assert_equal(mod.x, 9)
276 nt.assert_equal(mod.z, 123)
270 nt.assert_equal(mod.z, 123)
277
271
278 nt.assert_equal(old_foo(0), 3)
272 nt.assert_equal(old_foo(0), 3)
279 nt.assert_equal(mod.foo(0), 3)
273 nt.assert_equal(mod.foo(0), 3)
280
274
281 obj = mod.Baz(9)
275 obj = mod.Baz(9)
282 nt.assert_equal(old_obj.bar(1), 10)
276 nt.assert_equal(old_obj.bar(1), 10)
283 nt.assert_equal(obj.bar(1), 10)
277 nt.assert_equal(obj.bar(1), 10)
284 nt.assert_equal(obj.quux, 42)
278 nt.assert_equal(obj.quux, 42)
285 nt.assert_equal(obj.zzz(), 99)
279 nt.assert_equal(obj.zzz(), 99)
286
280
287 obj2 = mod.Bar()
281 obj2 = mod.Bar()
288 nt.assert_equal(old_obj2.foo(), 1)
282 nt.assert_equal(old_obj2.foo(), 1)
289 nt.assert_equal(obj2.foo(), 1)
283 nt.assert_equal(obj2.foo(), 1)
290
284
291 check_module_contents()
285 check_module_contents()
292
286
293 #
287 #
294 # Simulate a failed reload: no reload should occur and exactly
288 # Simulate a failed reload: no reload should occur and exactly
295 # one error message should be printed
289 # one error message should be printed
296 #
290 #
297 self.write_file(mod_fn, """
291 self.write_file(mod_fn, """
298 a syntax error
292 a syntax error
299 """)
293 """)
300
294
301 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
295 with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
302 self.shell.run_code("pass") # trigger reload
296 self.shell.run_code("pass") # trigger reload
303 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
297 with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
304 self.shell.run_code("pass") # trigger another reload
298 self.shell.run_code("pass") # trigger another reload
305 check_module_contents()
299 check_module_contents()
306
300
307 #
301 #
308 # Rewrite module (this time reload should succeed)
302 # Rewrite module (this time reload should succeed)
309 #
303 #
310 self.write_file(mod_fn, """
304 self.write_file(mod_fn, """
311 x = 10
305 x = 10
312
306
313 def foo(y):
307 def foo(y):
314 return y + 4
308 return y + 4
315
309
316 class Baz(object):
310 class Baz(object):
317 def __init__(self, x):
311 def __init__(self, x):
318 self.x = x
312 self.x = x
319 def bar(self, y):
313 def bar(self, y):
320 return self.x + y + 1
314 return self.x + y + 1
321 @property
315 @property
322 def quux(self):
316 def quux(self):
323 return 43
317 return 43
324
318
325 class Bar: # old-style class
319 class Bar: # old-style class
326 def foo(self):
320 def foo(self):
327 return 2
321 return 2
328 """)
322 """)
329
323
330 def check_module_contents():
324 def check_module_contents():
331 nt.assert_equal(mod.x, 10)
325 nt.assert_equal(mod.x, 10)
332 nt.assert_false(hasattr(mod, 'z'))
326 nt.assert_false(hasattr(mod, 'z'))
333
327
334 nt.assert_equal(old_foo(0), 4) # superreload magic!
328 nt.assert_equal(old_foo(0), 4) # superreload magic!
335 nt.assert_equal(mod.foo(0), 4)
329 nt.assert_equal(mod.foo(0), 4)
336
330
337 obj = mod.Baz(9)
331 obj = mod.Baz(9)
338 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
332 nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
339 nt.assert_equal(obj.bar(1), 11)
333 nt.assert_equal(obj.bar(1), 11)
340
334
341 nt.assert_equal(old_obj.quux, 43)
335 nt.assert_equal(old_obj.quux, 43)
342 nt.assert_equal(obj.quux, 43)
336 nt.assert_equal(obj.quux, 43)
343
337
344 nt.assert_false(hasattr(old_obj, 'zzz'))
338 nt.assert_false(hasattr(old_obj, 'zzz'))
345 nt.assert_false(hasattr(obj, 'zzz'))
339 nt.assert_false(hasattr(obj, 'zzz'))
346
340
347 obj2 = mod.Bar()
341 obj2 = mod.Bar()
348 nt.assert_equal(old_obj2.foo(), 2)
342 nt.assert_equal(old_obj2.foo(), 2)
349 nt.assert_equal(obj2.foo(), 2)
343 nt.assert_equal(obj2.foo(), 2)
350
344
351 self.shell.run_code("pass") # trigger reload
345 self.shell.run_code("pass") # trigger reload
352 check_module_contents()
346 check_module_contents()
353
347
354 #
348 #
355 # Another failure case: deleted file (shouldn't reload)
349 # Another failure case: deleted file (shouldn't reload)
356 #
350 #
357 os.unlink(mod_fn)
351 os.unlink(mod_fn)
358
352
359 self.shell.run_code("pass") # trigger reload
353 self.shell.run_code("pass") # trigger reload
360 check_module_contents()
354 check_module_contents()
361
355
362 #
356 #
363 # Disable autoreload and rewrite module: no reload should occur
357 # Disable autoreload and rewrite module: no reload should occur
364 #
358 #
365 if use_aimport:
359 if use_aimport:
366 self.shell.magic_aimport("-" + mod_name)
360 self.shell.magic_aimport("-" + mod_name)
367 stream = StringIO()
361 stream = StringIO()
368 self.shell.magic_aimport("", stream=stream)
362 self.shell.magic_aimport("", stream=stream)
369 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
363 nt.assert_true(("Modules to skip:\n%s" % mod_name) in
370 stream.getvalue())
364 stream.getvalue())
371
365
372 # This should succeed, although no such module exists
366 # This should succeed, although no such module exists
373 self.shell.magic_aimport("-tmpmod_as318989e89ds")
367 self.shell.magic_aimport("-tmpmod_as318989e89ds")
374 else:
368 else:
375 self.shell.magic_autoreload("0")
369 self.shell.magic_autoreload("0")
376
370
377 self.write_file(mod_fn, """
371 self.write_file(mod_fn, """
378 x = -99
372 x = -99
379 """)
373 """)
380
374
381 self.shell.run_code("pass") # trigger reload
375 self.shell.run_code("pass") # trigger reload
382 self.shell.run_code("pass")
376 self.shell.run_code("pass")
383 check_module_contents()
377 check_module_contents()
384
378
385 #
379 #
386 # Re-enable autoreload: reload should now occur
380 # Re-enable autoreload: reload should now occur
387 #
381 #
388 if use_aimport:
382 if use_aimport:
389 self.shell.magic_aimport(mod_name)
383 self.shell.magic_aimport(mod_name)
390 else:
384 else:
391 self.shell.magic_autoreload("")
385 self.shell.magic_autoreload("")
392
386
393 self.shell.run_code("pass") # trigger reload
387 self.shell.run_code("pass") # trigger reload
394 nt.assert_equal(mod.x, -99)
388 nt.assert_equal(mod.x, -99)
395
389
396 def test_smoketest_aimport(self):
390 def test_smoketest_aimport(self):
397 self._check_smoketest(use_aimport=True)
391 self._check_smoketest(use_aimport=True)
398
392
399 def test_smoketest_autoreload(self):
393 def test_smoketest_autoreload(self):
400 self._check_smoketest(use_aimport=False)
394 self._check_smoketest(use_aimport=False)
401
395
402
396
403
397
404
398
@@ -1,764 +1,761 b''
1 """Nose Plugin that supports IPython doctests.
1 """Nose Plugin that supports IPython doctests.
2
2
3 Limitations:
3 Limitations:
4
4
5 - When generating examples for use as doctests, make sure that you have
5 - When generating examples for use as doctests, make sure that you have
6 pretty-printing OFF. This can be done either by setting the
6 pretty-printing OFF. This can be done either by setting the
7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 by interactively disabling it with %Pprint. This is required so that IPython
8 by interactively disabling it with %Pprint. This is required so that IPython
9 output matches that of normal Python, which is used by doctest for internal
9 output matches that of normal Python, which is used by doctest for internal
10 execution.
10 execution.
11
11
12 - Do not rely on specific prompt numbers for results (such as using
12 - Do not rely on specific prompt numbers for results (such as using
13 '_34==True', for example). For IPython tests run via an external process the
13 '_34==True', for example). For IPython tests run via an external process the
14 prompt numbers may be different, and IPython tests run as normal python code
14 prompt numbers may be different, and IPython tests run as normal python code
15 won't even have these special _NN variables set at all.
15 won't even have these special _NN variables set at all.
16 """
16 """
17
17
18 #-----------------------------------------------------------------------------
18 #-----------------------------------------------------------------------------
19 # Module imports
19 # Module imports
20
20
21 # From the standard library
21 # From the standard library
22 import builtins as builtin_mod
22 import builtins as builtin_mod
23 import doctest
23 import doctest
24 import inspect
24 import inspect
25 import logging
25 import logging
26 import os
26 import os
27 import re
27 import re
28 import sys
28 import sys
29 from importlib import import_module
29 from importlib import import_module
30 from io import StringIO
30 from io import StringIO
31
31
32 from testpath import modified_env
32 from testpath import modified_env
33
33
34 from inspect import getmodule
34 from inspect import getmodule
35
35
36 # We are overriding the default doctest runner, so we need to import a few
36 # We are overriding the default doctest runner, so we need to import a few
37 # things from doctest directly
37 # things from doctest directly
38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
38 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
39 _unittest_reportflags, DocTestRunner,
39 _unittest_reportflags, DocTestRunner,
40 _extract_future_flags, pdb, _OutputRedirectingPdb,
40 _extract_future_flags, pdb, _OutputRedirectingPdb,
41 _exception_traceback,
41 _exception_traceback,
42 linecache)
42 linecache)
43
43
44 # Third-party modules
44 # Third-party modules
45
45
46 from nose.plugins import doctests, Plugin
46 from nose.plugins import doctests, Plugin
47 from nose.util import anyp, tolist
47 from nose.util import anyp, tolist
48
48
49 #-----------------------------------------------------------------------------
49 #-----------------------------------------------------------------------------
50 # Module globals and other constants
50 # Module globals and other constants
51 #-----------------------------------------------------------------------------
51 #-----------------------------------------------------------------------------
52
52
53 log = logging.getLogger(__name__)
53 log = logging.getLogger(__name__)
54
54
55
55
56 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
57 # Classes and functions
57 # Classes and functions
58 #-----------------------------------------------------------------------------
58 #-----------------------------------------------------------------------------
59
59
60 def is_extension_module(filename):
60 def is_extension_module(filename):
61 """Return whether the given filename is an extension module.
61 """Return whether the given filename is an extension module.
62
62
63 This simply checks that the extension is either .so or .pyd.
63 This simply checks that the extension is either .so or .pyd.
64 """
64 """
65 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
65 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
66
66
67
67
68 class DocTestSkip(object):
68 class DocTestSkip(object):
69 """Object wrapper for doctests to be skipped."""
69 """Object wrapper for doctests to be skipped."""
70
70
71 ds_skip = """Doctest to skip.
71 ds_skip = """Doctest to skip.
72 >>> 1 #doctest: +SKIP
72 >>> 1 #doctest: +SKIP
73 """
73 """
74
74
75 def __init__(self,obj):
75 def __init__(self,obj):
76 self.obj = obj
76 self.obj = obj
77
77
78 def __getattribute__(self,key):
78 def __getattribute__(self,key):
79 if key == '__doc__':
79 if key == '__doc__':
80 return DocTestSkip.ds_skip
80 return DocTestSkip.ds_skip
81 else:
81 else:
82 return getattr(object.__getattribute__(self,'obj'),key)
82 return getattr(object.__getattribute__(self,'obj'),key)
83
83
84 # Modified version of the one in the stdlib, that fixes a python bug (doctests
84 # Modified version of the one in the stdlib, that fixes a python bug (doctests
85 # not found in extension modules, http://bugs.python.org/issue3158)
85 # not found in extension modules, http://bugs.python.org/issue3158)
86 class DocTestFinder(doctest.DocTestFinder):
86 class DocTestFinder(doctest.DocTestFinder):
87
87
88 def _from_module(self, module, object):
88 def _from_module(self, module, object):
89 """
89 """
90 Return true if the given object is defined in the given
90 Return true if the given object is defined in the given
91 module.
91 module.
92 """
92 """
93 if module is None:
93 if module is None:
94 return True
94 return True
95 elif inspect.isfunction(object):
95 elif inspect.isfunction(object):
96 return module.__dict__ is object.__globals__
96 return module.__dict__ is object.__globals__
97 elif inspect.isbuiltin(object):
97 elif inspect.isbuiltin(object):
98 return module.__name__ == object.__module__
98 return module.__name__ == object.__module__
99 elif inspect.isclass(object):
99 elif inspect.isclass(object):
100 return module.__name__ == object.__module__
100 return module.__name__ == object.__module__
101 elif inspect.ismethod(object):
101 elif inspect.ismethod(object):
102 # This one may be a bug in cython that fails to correctly set the
102 # This one may be a bug in cython that fails to correctly set the
103 # __module__ attribute of methods, but since the same error is easy
103 # __module__ attribute of methods, but since the same error is easy
104 # to make by extension code writers, having this safety in place
104 # to make by extension code writers, having this safety in place
105 # isn't such a bad idea
105 # isn't such a bad idea
106 return module.__name__ == object.__self__.__class__.__module__
106 return module.__name__ == object.__self__.__class__.__module__
107 elif inspect.getmodule(object) is not None:
107 elif inspect.getmodule(object) is not None:
108 return module is inspect.getmodule(object)
108 return module is inspect.getmodule(object)
109 elif hasattr(object, '__module__'):
109 elif hasattr(object, '__module__'):
110 return module.__name__ == object.__module__
110 return module.__name__ == object.__module__
111 elif isinstance(object, property):
111 elif isinstance(object, property):
112 return True # [XX] no way not be sure.
112 return True # [XX] no way not be sure.
113 elif inspect.ismethoddescriptor(object):
113 elif inspect.ismethoddescriptor(object):
114 # Unbound PyQt signals reach this point in Python 3.4b3, and we want
114 # Unbound PyQt signals reach this point in Python 3.4b3, and we want
115 # to avoid throwing an error. See also http://bugs.python.org/issue3158
115 # to avoid throwing an error. See also http://bugs.python.org/issue3158
116 return False
116 return False
117 else:
117 else:
118 raise ValueError("object must be a class or function, got %r" % object)
118 raise ValueError("object must be a class or function, got %r" % object)
119
119
120 def _find(self, tests, obj, name, module, source_lines, globs, seen):
120 def _find(self, tests, obj, name, module, source_lines, globs, seen):
121 """
121 """
122 Find tests for the given object and any contained objects, and
122 Find tests for the given object and any contained objects, and
123 add them to `tests`.
123 add them to `tests`.
124 """
124 """
125 print('_find for:', obj, name, module) # dbg
125 print('_find for:', obj, name, module) # dbg
126 if hasattr(obj,"skip_doctest"):
126 if hasattr(obj,"skip_doctest"):
127 #print 'SKIPPING DOCTEST FOR:',obj # dbg
127 #print 'SKIPPING DOCTEST FOR:',obj # dbg
128 obj = DocTestSkip(obj)
128 obj = DocTestSkip(obj)
129
129
130 doctest.DocTestFinder._find(self,tests, obj, name, module,
130 doctest.DocTestFinder._find(self,tests, obj, name, module,
131 source_lines, globs, seen)
131 source_lines, globs, seen)
132
132
133 # Below we re-run pieces of the above method with manual modifications,
133 # Below we re-run pieces of the above method with manual modifications,
134 # because the original code is buggy and fails to correctly identify
134 # because the original code is buggy and fails to correctly identify
135 # doctests in extension modules.
135 # doctests in extension modules.
136
136
137 # Local shorthands
137 # Local shorthands
138 from inspect import isroutine, isclass
138 from inspect import isroutine, isclass
139
139
140 # Look for tests in a module's contained objects.
140 # Look for tests in a module's contained objects.
141 if inspect.ismodule(obj) and self._recurse:
141 if inspect.ismodule(obj) and self._recurse:
142 for valname, val in obj.__dict__.items():
142 for valname, val in obj.__dict__.items():
143 valname1 = '%s.%s' % (name, valname)
143 valname1 = '%s.%s' % (name, valname)
144 if ( (isroutine(val) or isclass(val))
144 if ( (isroutine(val) or isclass(val))
145 and self._from_module(module, val) ):
145 and self._from_module(module, val) ):
146
146
147 self._find(tests, val, valname1, module, source_lines,
147 self._find(tests, val, valname1, module, source_lines,
148 globs, seen)
148 globs, seen)
149
149
150 # Look for tests in a class's contained objects.
150 # Look for tests in a class's contained objects.
151 if inspect.isclass(obj) and self._recurse:
151 if inspect.isclass(obj) and self._recurse:
152 #print 'RECURSE into class:',obj # dbg
152 #print 'RECURSE into class:',obj # dbg
153 for valname, val in obj.__dict__.items():
153 for valname, val in obj.__dict__.items():
154 # Special handling for staticmethod/classmethod.
154 # Special handling for staticmethod/classmethod.
155 if isinstance(val, staticmethod):
155 if isinstance(val, staticmethod):
156 val = getattr(obj, valname)
156 val = getattr(obj, valname)
157 if isinstance(val, classmethod):
157 if isinstance(val, classmethod):
158 val = getattr(obj, valname).__func__
158 val = getattr(obj, valname).__func__
159
159
160 # Recurse to methods, properties, and nested classes.
160 # Recurse to methods, properties, and nested classes.
161 if ((inspect.isfunction(val) or inspect.isclass(val) or
161 if ((inspect.isfunction(val) or inspect.isclass(val) or
162 inspect.ismethod(val) or
162 inspect.ismethod(val) or
163 isinstance(val, property)) and
163 isinstance(val, property)) and
164 self._from_module(module, val)):
164 self._from_module(module, val)):
165 valname = '%s.%s' % (name, valname)
165 valname = '%s.%s' % (name, valname)
166 self._find(tests, val, valname, module, source_lines,
166 self._find(tests, val, valname, module, source_lines,
167 globs, seen)
167 globs, seen)
168
168
169
169
170 class IPDoctestOutputChecker(doctest.OutputChecker):
170 class IPDoctestOutputChecker(doctest.OutputChecker):
171 """Second-chance checker with support for random tests.
171 """Second-chance checker with support for random tests.
172
172
173 If the default comparison doesn't pass, this checker looks in the expected
173 If the default comparison doesn't pass, this checker looks in the expected
174 output string for flags that tell us to ignore the output.
174 output string for flags that tell us to ignore the output.
175 """
175 """
176
176
177 random_re = re.compile(r'#\s*random\s+')
177 random_re = re.compile(r'#\s*random\s+')
178
178
179 def check_output(self, want, got, optionflags):
179 def check_output(self, want, got, optionflags):
180 """Check output, accepting special markers embedded in the output.
180 """Check output, accepting special markers embedded in the output.
181
181
182 If the output didn't pass the default validation but the special string
182 If the output didn't pass the default validation but the special string
183 '#random' is included, we accept it."""
183 '#random' is included, we accept it."""
184
184
185 # Let the original tester verify first, in case people have valid tests
185 # Let the original tester verify first, in case people have valid tests
186 # that happen to have a comment saying '#random' embedded in.
186 # that happen to have a comment saying '#random' embedded in.
187 ret = doctest.OutputChecker.check_output(self, want, got,
187 ret = doctest.OutputChecker.check_output(self, want, got,
188 optionflags)
188 optionflags)
189 if not ret and self.random_re.search(want):
189 if not ret and self.random_re.search(want):
190 #print >> sys.stderr, 'RANDOM OK:',want # dbg
190 #print >> sys.stderr, 'RANDOM OK:',want # dbg
191 return True
191 return True
192
192
193 return ret
193 return ret
194
194
195
195
196 class DocTestCase(doctests.DocTestCase):
196 class DocTestCase(doctests.DocTestCase):
197 """Proxy for DocTestCase: provides an address() method that
197 """Proxy for DocTestCase: provides an address() method that
198 returns the correct address for the doctest case. Otherwise
198 returns the correct address for the doctest case. Otherwise
199 acts as a proxy to the test case. To provide hints for address(),
199 acts as a proxy to the test case. To provide hints for address(),
200 an obj may also be passed -- this will be used as the test object
200 an obj may also be passed -- this will be used as the test object
201 for purposes of determining the test address, if it is provided.
201 for purposes of determining the test address, if it is provided.
202 """
202 """
203
203
204 # Note: this method was taken from numpy's nosetester module.
204 # Note: this method was taken from numpy's nosetester module.
205
205
206 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
206 # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
207 # its constructor that blocks non-default arguments from being passed
207 # its constructor that blocks non-default arguments from being passed
208 # down into doctest.DocTestCase
208 # down into doctest.DocTestCase
209
209
210 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
210 def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
211 checker=None, obj=None, result_var='_'):
211 checker=None, obj=None, result_var='_'):
212 self._result_var = result_var
212 self._result_var = result_var
213 doctests.DocTestCase.__init__(self, test,
213 doctests.DocTestCase.__init__(self, test,
214 optionflags=optionflags,
214 optionflags=optionflags,
215 setUp=setUp, tearDown=tearDown,
215 setUp=setUp, tearDown=tearDown,
216 checker=checker)
216 checker=checker)
217 # Now we must actually copy the original constructor from the stdlib
217 # Now we must actually copy the original constructor from the stdlib
218 # doctest class, because we can't call it directly and a bug in nose
218 # doctest class, because we can't call it directly and a bug in nose
219 # means it never gets passed the right arguments.
219 # means it never gets passed the right arguments.
220
220
221 self._dt_optionflags = optionflags
221 self._dt_optionflags = optionflags
222 self._dt_checker = checker
222 self._dt_checker = checker
223 self._dt_test = test
223 self._dt_test = test
224 self._dt_test_globs_ori = test.globs
224 self._dt_test_globs_ori = test.globs
225 self._dt_setUp = setUp
225 self._dt_setUp = setUp
226 self._dt_tearDown = tearDown
226 self._dt_tearDown = tearDown
227
227
228 # XXX - store this runner once in the object!
228 # XXX - store this runner once in the object!
229 runner = IPDocTestRunner(optionflags=optionflags,
229 runner = IPDocTestRunner(optionflags=optionflags,
230 checker=checker, verbose=False)
230 checker=checker, verbose=False)
231 self._dt_runner = runner
231 self._dt_runner = runner
232
232
233
233
234 # Each doctest should remember the directory it was loaded from, so
234 # Each doctest should remember the directory it was loaded from, so
235 # things like %run work without too many contortions
235 # things like %run work without too many contortions
236 self._ori_dir = os.path.dirname(test.filename)
236 self._ori_dir = os.path.dirname(test.filename)
237
237
238 # Modified runTest from the default stdlib
238 # Modified runTest from the default stdlib
239 def runTest(self):
239 def runTest(self):
240 test = self._dt_test
240 test = self._dt_test
241 runner = self._dt_runner
241 runner = self._dt_runner
242
242
243 old = sys.stdout
243 old = sys.stdout
244 new = StringIO()
244 new = StringIO()
245 optionflags = self._dt_optionflags
245 optionflags = self._dt_optionflags
246
246
247 if not (optionflags & REPORTING_FLAGS):
247 if not (optionflags & REPORTING_FLAGS):
248 # The option flags don't include any reporting flags,
248 # The option flags don't include any reporting flags,
249 # so add the default reporting flags
249 # so add the default reporting flags
250 optionflags |= _unittest_reportflags
250 optionflags |= _unittest_reportflags
251
251
252 try:
252 try:
253 # Save our current directory and switch out to the one where the
253 # Save our current directory and switch out to the one where the
254 # test was originally created, in case another doctest did a
254 # test was originally created, in case another doctest did a
255 # directory change. We'll restore this in the finally clause.
255 # directory change. We'll restore this in the finally clause.
256 curdir = os.getcwd()
256 curdir = os.getcwd()
257 #print 'runTest in dir:', self._ori_dir # dbg
257 #print 'runTest in dir:', self._ori_dir # dbg
258 os.chdir(self._ori_dir)
258 os.chdir(self._ori_dir)
259
259
260 runner.DIVIDER = "-"*70
260 runner.DIVIDER = "-"*70
261 failures, tries = runner.run(test,out=new.write,
261 failures, tries = runner.run(test,out=new.write,
262 clear_globs=False)
262 clear_globs=False)
263 finally:
263 finally:
264 sys.stdout = old
264 sys.stdout = old
265 os.chdir(curdir)
265 os.chdir(curdir)
266
266
267 if failures:
267 if failures:
268 raise self.failureException(self.format_failure(new.getvalue()))
268 raise self.failureException(self.format_failure(new.getvalue()))
269
269
270 def setUp(self):
270 def setUp(self):
271 """Modified test setup that syncs with ipython namespace"""
271 """Modified test setup that syncs with ipython namespace"""
272 #print "setUp test", self._dt_test.examples # dbg
272 #print "setUp test", self._dt_test.examples # dbg
273 if isinstance(self._dt_test.examples[0], IPExample):
273 if isinstance(self._dt_test.examples[0], IPExample):
274 # for IPython examples *only*, we swap the globals with the ipython
274 # for IPython examples *only*, we swap the globals with the ipython
275 # namespace, after updating it with the globals (which doctest
275 # namespace, after updating it with the globals (which doctest
276 # fills with the necessary info from the module being tested).
276 # fills with the necessary info from the module being tested).
277 self.user_ns_orig = {}
277 self.user_ns_orig = {}
278 self.user_ns_orig.update(_ip.user_ns)
278 self.user_ns_orig.update(_ip.user_ns)
279 _ip.user_ns.update(self._dt_test.globs)
279 _ip.user_ns.update(self._dt_test.globs)
280 # We must remove the _ key in the namespace, so that Python's
280 # We must remove the _ key in the namespace, so that Python's
281 # doctest code sets it naturally
281 # doctest code sets it naturally
282 _ip.user_ns.pop('_', None)
282 _ip.user_ns.pop('_', None)
283 _ip.user_ns['__builtins__'] = builtin_mod
283 _ip.user_ns['__builtins__'] = builtin_mod
284 self._dt_test.globs = _ip.user_ns
284 self._dt_test.globs = _ip.user_ns
285
285
286 super(DocTestCase, self).setUp()
286 super(DocTestCase, self).setUp()
287
287
288 def tearDown(self):
288 def tearDown(self):
289
289
290 # Undo the test.globs reassignment we made, so that the parent class
290 # Undo the test.globs reassignment we made, so that the parent class
291 # teardown doesn't destroy the ipython namespace
291 # teardown doesn't destroy the ipython namespace
292 if isinstance(self._dt_test.examples[0], IPExample):
292 if isinstance(self._dt_test.examples[0], IPExample):
293 self._dt_test.globs = self._dt_test_globs_ori
293 self._dt_test.globs = self._dt_test_globs_ori
294 _ip.user_ns.clear()
294 _ip.user_ns.clear()
295 _ip.user_ns.update(self.user_ns_orig)
295 _ip.user_ns.update(self.user_ns_orig)
296
296
297 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
297 # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
298 # it does look like one to me: its tearDown method tries to run
298 # it does look like one to me: its tearDown method tries to run
299 #
299 #
300 # delattr(builtin_mod, self._result_var)
300 # delattr(builtin_mod, self._result_var)
301 #
301 #
302 # without checking that the attribute really is there; it implicitly
302 # without checking that the attribute really is there; it implicitly
303 # assumes it should have been set via displayhook. But if the
303 # assumes it should have been set via displayhook. But if the
304 # displayhook was never called, this doesn't necessarily happen. I
304 # displayhook was never called, this doesn't necessarily happen. I
305 # haven't been able to find a little self-contained example outside of
305 # haven't been able to find a little self-contained example outside of
306 # ipython that would show the problem so I can report it to the nose
306 # ipython that would show the problem so I can report it to the nose
307 # team, but it does happen a lot in our code.
307 # team, but it does happen a lot in our code.
308 #
308 #
309 # So here, we just protect as narrowly as possible by trapping an
309 # So here, we just protect as narrowly as possible by trapping an
310 # attribute error whose message would be the name of self._result_var,
310 # attribute error whose message would be the name of self._result_var,
311 # and letting any other error propagate.
311 # and letting any other error propagate.
312 try:
312 try:
313 super(DocTestCase, self).tearDown()
313 super(DocTestCase, self).tearDown()
314 except AttributeError as exc:
314 except AttributeError as exc:
315 if exc.args[0] != self._result_var:
315 if exc.args[0] != self._result_var:
316 raise
316 raise
317
317
318
318
319 # A simple subclassing of the original with a different class name, so we can
319 # A simple subclassing of the original with a different class name, so we can
320 # distinguish and treat differently IPython examples from pure python ones.
320 # distinguish and treat differently IPython examples from pure python ones.
321 class IPExample(doctest.Example): pass
321 class IPExample(doctest.Example): pass
322
322
323
323
324 class IPExternalExample(doctest.Example):
324 class IPExternalExample(doctest.Example):
325 """Doctest examples to be run in an external process."""
325 """Doctest examples to be run in an external process."""
326
326
327 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
327 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
328 options=None):
328 options=None):
329 # Parent constructor
329 # Parent constructor
330 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
330 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
331
331
332 # An EXTRA newline is needed to prevent pexpect hangs
332 # An EXTRA newline is needed to prevent pexpect hangs
333 self.source += '\n'
333 self.source += '\n'
334
334
335
335
336 class IPDocTestParser(doctest.DocTestParser):
336 class IPDocTestParser(doctest.DocTestParser):
337 """
337 """
338 A class used to parse strings containing doctest examples.
338 A class used to parse strings containing doctest examples.
339
339
340 Note: This is a version modified to properly recognize IPython input and
340 Note: This is a version modified to properly recognize IPython input and
341 convert any IPython examples into valid Python ones.
341 convert any IPython examples into valid Python ones.
342 """
342 """
343 # This regular expression is used to find doctest examples in a
343 # This regular expression is used to find doctest examples in a
344 # string. It defines three groups: `source` is the source code
344 # string. It defines three groups: `source` is the source code
345 # (including leading indentation and prompts); `indent` is the
345 # (including leading indentation and prompts); `indent` is the
346 # indentation of the first (PS1) line of the source code; and
346 # indentation of the first (PS1) line of the source code; and
347 # `want` is the expected output (including leading indentation).
347 # `want` is the expected output (including leading indentation).
348
348
349 # Classic Python prompts or default IPython ones
349 # Classic Python prompts or default IPython ones
350 _PS1_PY = r'>>>'
350 _PS1_PY = r'>>>'
351 _PS2_PY = r'\.\.\.'
351 _PS2_PY = r'\.\.\.'
352
352
353 _PS1_IP = r'In\ \[\d+\]:'
353 _PS1_IP = r'In\ \[\d+\]:'
354 _PS2_IP = r'\ \ \ \.\.\.+:'
354 _PS2_IP = r'\ \ \ \.\.\.+:'
355
355
356 _RE_TPL = r'''
356 _RE_TPL = r'''
357 # Source consists of a PS1 line followed by zero or more PS2 lines.
357 # Source consists of a PS1 line followed by zero or more PS2 lines.
358 (?P<source>
358 (?P<source>
359 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
359 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
360 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
360 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
361 \n? # a newline
361 \n? # a newline
362 # Want consists of any non-blank lines that do not start with PS1.
362 # Want consists of any non-blank lines that do not start with PS1.
363 (?P<want> (?:(?![ ]*$) # Not a blank line
363 (?P<want> (?:(?![ ]*$) # Not a blank line
364 (?![ ]*%s) # Not a line starting with PS1
364 (?![ ]*%s) # Not a line starting with PS1
365 (?![ ]*%s) # Not a line starting with PS2
365 (?![ ]*%s) # Not a line starting with PS2
366 .*$\n? # But any other line
366 .*$\n? # But any other line
367 )*)
367 )*)
368 '''
368 '''
369
369
370 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
370 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
371 re.MULTILINE | re.VERBOSE)
371 re.MULTILINE | re.VERBOSE)
372
372
373 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
373 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
374 re.MULTILINE | re.VERBOSE)
374 re.MULTILINE | re.VERBOSE)
375
375
376 # Mark a test as being fully random. In this case, we simply append the
376 # Mark a test as being fully random. In this case, we simply append the
377 # random marker ('#random') to each individual example's output. This way
377 # random marker ('#random') to each individual example's output. This way
378 # we don't need to modify any other code.
378 # we don't need to modify any other code.
379 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
379 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
380
380
381 # Mark tests to be executed in an external process - currently unsupported.
381 # Mark tests to be executed in an external process - currently unsupported.
382 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
382 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
383
383
384 def ip2py(self,source):
384 def ip2py(self,source):
385 """Convert input IPython source into valid Python."""
385 """Convert input IPython source into valid Python."""
386 block = _ip.input_transformer_manager.transform_cell(source)
386 block = _ip.input_transformer_manager.transform_cell(source)
387 if len(block.splitlines()) == 1:
387 if len(block.splitlines()) == 1:
388 return _ip.prefilter(block)
388 return _ip.prefilter(block)
389 else:
389 else:
390 return block
390 return block
391
391
392 def parse(self, string, name='<string>'):
392 def parse(self, string, name='<string>'):
393 """
393 """
394 Divide the given string into examples and intervening text,
394 Divide the given string into examples and intervening text,
395 and return them as a list of alternating Examples and strings.
395 and return them as a list of alternating Examples and strings.
396 Line numbers for the Examples are 0-based. The optional
396 Line numbers for the Examples are 0-based. The optional
397 argument `name` is a name identifying this string, and is only
397 argument `name` is a name identifying this string, and is only
398 used for error messages.
398 used for error messages.
399 """
399 """
400
400
401 #print 'Parse string:\n',string # dbg
401 #print 'Parse string:\n',string # dbg
402
402
403 string = string.expandtabs()
403 string = string.expandtabs()
404 # If all lines begin with the same indentation, then strip it.
404 # If all lines begin with the same indentation, then strip it.
405 min_indent = self._min_indent(string)
405 min_indent = self._min_indent(string)
406 if min_indent > 0:
406 if min_indent > 0:
407 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
407 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
408
408
409 output = []
409 output = []
410 charno, lineno = 0, 0
410 charno, lineno = 0, 0
411
411
412 # We make 'all random' tests by adding the '# random' mark to every
412 # We make 'all random' tests by adding the '# random' mark to every
413 # block of output in the test.
413 # block of output in the test.
414 if self._RANDOM_TEST.search(string):
414 if self._RANDOM_TEST.search(string):
415 random_marker = '\n# random'
415 random_marker = '\n# random'
416 else:
416 else:
417 random_marker = ''
417 random_marker = ''
418
418
419 # Whether to convert the input from ipython to python syntax
419 # Whether to convert the input from ipython to python syntax
420 ip2py = False
420 ip2py = False
421 # Find all doctest examples in the string. First, try them as Python
421 # Find all doctest examples in the string. First, try them as Python
422 # examples, then as IPython ones
422 # examples, then as IPython ones
423 terms = list(self._EXAMPLE_RE_PY.finditer(string))
423 terms = list(self._EXAMPLE_RE_PY.finditer(string))
424 if terms:
424 if terms:
425 # Normal Python example
425 # Normal Python example
426 #print '-'*70 # dbg
426 #print '-'*70 # dbg
427 #print 'PyExample, Source:\n',string # dbg
427 #print 'PyExample, Source:\n',string # dbg
428 #print '-'*70 # dbg
428 #print '-'*70 # dbg
429 Example = doctest.Example
429 Example = doctest.Example
430 else:
430 else:
431 # It's an ipython example. Note that IPExamples are run
431 # It's an ipython example. Note that IPExamples are run
432 # in-process, so their syntax must be turned into valid python.
432 # in-process, so their syntax must be turned into valid python.
433 # IPExternalExamples are run out-of-process (via pexpect) so they
433 # IPExternalExamples are run out-of-process (via pexpect) so they
434 # don't need any filtering (a real ipython will be executing them).
434 # don't need any filtering (a real ipython will be executing them).
435 terms = list(self._EXAMPLE_RE_IP.finditer(string))
435 terms = list(self._EXAMPLE_RE_IP.finditer(string))
436 if self._EXTERNAL_IP.search(string):
436 if self._EXTERNAL_IP.search(string):
437 #print '-'*70 # dbg
437 #print '-'*70 # dbg
438 #print 'IPExternalExample, Source:\n',string # dbg
438 #print 'IPExternalExample, Source:\n',string # dbg
439 #print '-'*70 # dbg
439 #print '-'*70 # dbg
440 Example = IPExternalExample
440 Example = IPExternalExample
441 else:
441 else:
442 #print '-'*70 # dbg
442 #print '-'*70 # dbg
443 #print 'IPExample, Source:\n',string # dbg
443 #print 'IPExample, Source:\n',string # dbg
444 #print '-'*70 # dbg
444 #print '-'*70 # dbg
445 Example = IPExample
445 Example = IPExample
446 ip2py = True
446 ip2py = True
447
447
448 for m in terms:
448 for m in terms:
449 # Add the pre-example text to `output`.
449 # Add the pre-example text to `output`.
450 output.append(string[charno:m.start()])
450 output.append(string[charno:m.start()])
451 # Update lineno (lines before this example)
451 # Update lineno (lines before this example)
452 lineno += string.count('\n', charno, m.start())
452 lineno += string.count('\n', charno, m.start())
453 # Extract info from the regexp match.
453 # Extract info from the regexp match.
454 (source, options, want, exc_msg) = \
454 (source, options, want, exc_msg) = \
455 self._parse_example(m, name, lineno,ip2py)
455 self._parse_example(m, name, lineno,ip2py)
456
456
457 # Append the random-output marker (it defaults to empty in most
457 # Append the random-output marker (it defaults to empty in most
458 # cases, it's only non-empty for 'all-random' tests):
458 # cases, it's only non-empty for 'all-random' tests):
459 want += random_marker
459 want += random_marker
460
460
461 if Example is IPExternalExample:
461 if Example is IPExternalExample:
462 options[doctest.NORMALIZE_WHITESPACE] = True
462 options[doctest.NORMALIZE_WHITESPACE] = True
463 want += '\n'
463 want += '\n'
464
464
465 # Create an Example, and add it to the list.
465 # Create an Example, and add it to the list.
466 if not self._IS_BLANK_OR_COMMENT(source):
466 if not self._IS_BLANK_OR_COMMENT(source):
467 output.append(Example(source, want, exc_msg,
467 output.append(Example(source, want, exc_msg,
468 lineno=lineno,
468 lineno=lineno,
469 indent=min_indent+len(m.group('indent')),
469 indent=min_indent+len(m.group('indent')),
470 options=options))
470 options=options))
471 # Update lineno (lines inside this example)
471 # Update lineno (lines inside this example)
472 lineno += string.count('\n', m.start(), m.end())
472 lineno += string.count('\n', m.start(), m.end())
473 # Update charno.
473 # Update charno.
474 charno = m.end()
474 charno = m.end()
475 # Add any remaining post-example text to `output`.
475 # Add any remaining post-example text to `output`.
476 output.append(string[charno:])
476 output.append(string[charno:])
477 return output
477 return output
478
478
479 def _parse_example(self, m, name, lineno,ip2py=False):
479 def _parse_example(self, m, name, lineno,ip2py=False):
480 """
480 """
481 Given a regular expression match from `_EXAMPLE_RE` (`m`),
481 Given a regular expression match from `_EXAMPLE_RE` (`m`),
482 return a pair `(source, want)`, where `source` is the matched
482 return a pair `(source, want)`, where `source` is the matched
483 example's source code (with prompts and indentation stripped);
483 example's source code (with prompts and indentation stripped);
484 and `want` is the example's expected output (with indentation
484 and `want` is the example's expected output (with indentation
485 stripped).
485 stripped).
486
486
487 `name` is the string's name, and `lineno` is the line number
487 `name` is the string's name, and `lineno` is the line number
488 where the example starts; both are used for error messages.
488 where the example starts; both are used for error messages.
489
489
490 Optional:
490 Optional:
491 `ip2py`: if true, filter the input via IPython to convert the syntax
491 `ip2py`: if true, filter the input via IPython to convert the syntax
492 into valid python.
492 into valid python.
493 """
493 """
494
494
495 # Get the example's indentation level.
495 # Get the example's indentation level.
496 indent = len(m.group('indent'))
496 indent = len(m.group('indent'))
497
497
498 # Divide source into lines; check that they're properly
498 # Divide source into lines; check that they're properly
499 # indented; and then strip their indentation & prompts.
499 # indented; and then strip their indentation & prompts.
500 source_lines = m.group('source').split('\n')
500 source_lines = m.group('source').split('\n')
501
501
502 # We're using variable-length input prompts
502 # We're using variable-length input prompts
503 ps1 = m.group('ps1')
503 ps1 = m.group('ps1')
504 ps2 = m.group('ps2')
504 ps2 = m.group('ps2')
505 ps1_len = len(ps1)
505 ps1_len = len(ps1)
506
506
507 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
507 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
508 if ps2:
508 if ps2:
509 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
509 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
510
510
511 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
511 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
512
512
513 if ip2py:
513 if ip2py:
514 # Convert source input from IPython into valid Python syntax
514 # Convert source input from IPython into valid Python syntax
515 source = self.ip2py(source)
515 source = self.ip2py(source)
516
516
517 # Divide want into lines; check that it's properly indented; and
517 # Divide want into lines; check that it's properly indented; and
518 # then strip the indentation. Spaces before the last newline should
518 # then strip the indentation. Spaces before the last newline should
519 # be preserved, so plain rstrip() isn't good enough.
519 # be preserved, so plain rstrip() isn't good enough.
520 want = m.group('want')
520 want = m.group('want')
521 want_lines = want.split('\n')
521 want_lines = want.split('\n')
522 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
522 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
523 del want_lines[-1] # forget final newline & spaces after it
523 del want_lines[-1] # forget final newline & spaces after it
524 self._check_prefix(want_lines, ' '*indent, name,
524 self._check_prefix(want_lines, ' '*indent, name,
525 lineno + len(source_lines))
525 lineno + len(source_lines))
526
526
527 # Remove ipython output prompt that might be present in the first line
527 # Remove ipython output prompt that might be present in the first line
528 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
528 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
529
529
530 want = '\n'.join([wl[indent:] for wl in want_lines])
530 want = '\n'.join([wl[indent:] for wl in want_lines])
531
531
532 # If `want` contains a traceback message, then extract it.
532 # If `want` contains a traceback message, then extract it.
533 m = self._EXCEPTION_RE.match(want)
533 m = self._EXCEPTION_RE.match(want)
534 if m:
534 if m:
535 exc_msg = m.group('msg')
535 exc_msg = m.group('msg')
536 else:
536 else:
537 exc_msg = None
537 exc_msg = None
538
538
539 # Extract options from the source.
539 # Extract options from the source.
540 options = self._find_options(source, name, lineno)
540 options = self._find_options(source, name, lineno)
541
541
542 return source, options, want, exc_msg
542 return source, options, want, exc_msg
543
543
544 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
544 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
545 """
545 """
546 Given the lines of a source string (including prompts and
546 Given the lines of a source string (including prompts and
547 leading indentation), check to make sure that every prompt is
547 leading indentation), check to make sure that every prompt is
548 followed by a space character. If any line is not followed by
548 followed by a space character. If any line is not followed by
549 a space character, then raise ValueError.
549 a space character, then raise ValueError.
550
550
551 Note: IPython-modified version which takes the input prompt length as a
551 Note: IPython-modified version which takes the input prompt length as a
552 parameter, so that prompts of variable length can be dealt with.
552 parameter, so that prompts of variable length can be dealt with.
553 """
553 """
554 space_idx = indent+ps1_len
554 space_idx = indent+ps1_len
555 min_len = space_idx+1
555 min_len = space_idx+1
556 for i, line in enumerate(lines):
556 for i, line in enumerate(lines):
557 if len(line) >= min_len and line[space_idx] != ' ':
557 if len(line) >= min_len and line[space_idx] != ' ':
558 raise ValueError('line %r of the docstring for %s '
558 raise ValueError('line %r of the docstring for %s '
559 'lacks blank after %s: %r' %
559 'lacks blank after %s: %r' %
560 (lineno+i+1, name,
560 (lineno+i+1, name,
561 line[indent:space_idx], line))
561 line[indent:space_idx], line))
562
562
563
563
564 SKIP = doctest.register_optionflag('SKIP')
564 SKIP = doctest.register_optionflag('SKIP')
565
565
566
566
567 class IPDocTestRunner(doctest.DocTestRunner,object):
567 class IPDocTestRunner(doctest.DocTestRunner,object):
568 """Test runner that synchronizes the IPython namespace with test globals.
568 """Test runner that synchronizes the IPython namespace with test globals.
569 """
569 """
570
570
571 def run(self, test, compileflags=None, out=None, clear_globs=True):
571 def run(self, test, compileflags=None, out=None, clear_globs=True):
572
572
573 # Hack: ipython needs access to the execution context of the example,
573 # Hack: ipython needs access to the execution context of the example,
574 # so that it can propagate user variables loaded by %run into
574 # so that it can propagate user variables loaded by %run into
575 # test.globs. We put them here into our modified %run as a function
575 # test.globs. We put them here into our modified %run as a function
576 # attribute. Our new %run will then only make the namespace update
576 # attribute. Our new %run will then only make the namespace update
577 # when called (rather than unconconditionally updating test.globs here
577 # when called (rather than unconconditionally updating test.globs here
578 # for all examples, most of which won't be calling %run anyway).
578 # for all examples, most of which won't be calling %run anyway).
579 #_ip._ipdoctest_test_globs = test.globs
579 #_ip._ipdoctest_test_globs = test.globs
580 #_ip._ipdoctest_test_filename = test.filename
580 #_ip._ipdoctest_test_filename = test.filename
581
581
582 test.globs.update(_ip.user_ns)
582 test.globs.update(_ip.user_ns)
583
583
584 # Override terminal size to standardise traceback format
584 # Override terminal size to standardise traceback format
585 with modified_env({'COLUMNS': '80', 'LINES': '24'}):
585 with modified_env({'COLUMNS': '80', 'LINES': '24'}):
586 return super(IPDocTestRunner,self).run(test,
586 return super(IPDocTestRunner,self).run(test,
587 compileflags,out,clear_globs)
587 compileflags,out,clear_globs)
588
588
589
589
590 class DocFileCase(doctest.DocFileCase):
590 class DocFileCase(doctest.DocFileCase):
591 """Overrides to provide filename
591 """Overrides to provide filename
592 """
592 """
593 def address(self):
593 def address(self):
594 return (self._dt_test.filename, None, None)
594 return (self._dt_test.filename, None, None)
595
595
596
596
597 class ExtensionDoctest(doctests.Doctest):
597 class ExtensionDoctest(doctests.Doctest):
598 """Nose Plugin that supports doctests in extension modules.
598 """Nose Plugin that supports doctests in extension modules.
599 """
599 """
600 name = 'extdoctest' # call nosetests with --with-extdoctest
600 name = 'extdoctest' # call nosetests with --with-extdoctest
601 enabled = True
601 enabled = True
602
602
603 def options(self, parser, env=os.environ):
603 def options(self, parser, env=os.environ):
604 Plugin.options(self, parser, env)
604 Plugin.options(self, parser, env)
605 parser.add_option('--doctest-tests', action='store_true',
605 parser.add_option('--doctest-tests', action='store_true',
606 dest='doctest_tests',
606 dest='doctest_tests',
607 default=env.get('NOSE_DOCTEST_TESTS',True),
607 default=env.get('NOSE_DOCTEST_TESTS',True),
608 help="Also look for doctests in test modules. "
608 help="Also look for doctests in test modules. "
609 "Note that classes, methods and functions should "
609 "Note that classes, methods and functions should "
610 "have either doctests or non-doctest tests, "
610 "have either doctests or non-doctest tests, "
611 "not both. [NOSE_DOCTEST_TESTS]")
611 "not both. [NOSE_DOCTEST_TESTS]")
612 parser.add_option('--doctest-extension', action="append",
612 parser.add_option('--doctest-extension', action="append",
613 dest="doctestExtension",
613 dest="doctestExtension",
614 help="Also look for doctests in files with "
614 help="Also look for doctests in files with "
615 "this extension [NOSE_DOCTEST_EXTENSION]")
615 "this extension [NOSE_DOCTEST_EXTENSION]")
616 # Set the default as a list, if given in env; otherwise
616 # Set the default as a list, if given in env; otherwise
617 # an additional value set on the command line will cause
617 # an additional value set on the command line will cause
618 # an error.
618 # an error.
619 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
619 env_setting = env.get('NOSE_DOCTEST_EXTENSION')
620 if env_setting is not None:
620 if env_setting is not None:
621 parser.set_defaults(doctestExtension=tolist(env_setting))
621 parser.set_defaults(doctestExtension=tolist(env_setting))
622
622
623
623
624 def configure(self, options, config):
624 def configure(self, options, config):
625 Plugin.configure(self, options, config)
625 Plugin.configure(self, options, config)
626 # Pull standard doctest plugin out of config; we will do doctesting
626 # Pull standard doctest plugin out of config; we will do doctesting
627 config.plugins.plugins = [p for p in config.plugins.plugins
627 config.plugins.plugins = [p for p in config.plugins.plugins
628 if p.name != 'doctest']
628 if p.name != 'doctest']
629 self.doctest_tests = options.doctest_tests
629 self.doctest_tests = options.doctest_tests
630 self.extension = tolist(options.doctestExtension)
630 self.extension = tolist(options.doctestExtension)
631
631
632 self.parser = doctest.DocTestParser()
632 self.parser = doctest.DocTestParser()
633 self.finder = DocTestFinder()
633 self.finder = DocTestFinder()
634 self.checker = IPDoctestOutputChecker()
634 self.checker = IPDoctestOutputChecker()
635 self.globs = None
635 self.globs = None
636 self.extraglobs = None
636 self.extraglobs = None
637
637
638
638
639 def loadTestsFromExtensionModule(self,filename):
639 def loadTestsFromExtensionModule(self,filename):
640 bpath,mod = os.path.split(filename)
640 bpath,mod = os.path.split(filename)
641 modname = os.path.splitext(mod)[0]
641 modname = os.path.splitext(mod)[0]
642 try:
642 try:
643 sys.path.append(bpath)
643 sys.path.append(bpath)
644 module = import_module(modname)
644 module = import_module(modname)
645 tests = list(self.loadTestsFromModule(module))
645 tests = list(self.loadTestsFromModule(module))
646 finally:
646 finally:
647 sys.path.pop()
647 sys.path.pop()
648 return tests
648 return tests
649
649
650 # NOTE: the method below is almost a copy of the original one in nose, with
650 # NOTE: the method below is almost a copy of the original one in nose, with
651 # a few modifications to control output checking.
651 # a few modifications to control output checking.
652
652
653 def loadTestsFromModule(self, module):
653 def loadTestsFromModule(self, module):
654 #print '*** ipdoctest - lTM',module # dbg
654 #print '*** ipdoctest - lTM',module # dbg
655
655
656 if not self.matches(module.__name__):
656 if not self.matches(module.__name__):
657 log.debug("Doctest doesn't want module %s", module)
657 log.debug("Doctest doesn't want module %s", module)
658 return
658 return
659
659
660 tests = self.finder.find(module,globs=self.globs,
660 tests = self.finder.find(module,globs=self.globs,
661 extraglobs=self.extraglobs)
661 extraglobs=self.extraglobs)
662 if not tests:
662 if not tests:
663 return
663 return
664
664
665 # always use whitespace and ellipsis options
665 # always use whitespace and ellipsis options
666 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
666 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
667
667
668 tests.sort()
668 tests.sort()
669 module_file = module.__file__
669 module_file = module.__file__
670 if module_file[-4:] in ('.pyc', '.pyo'):
670 if module_file[-4:] in ('.pyc', '.pyo'):
671 module_file = module_file[:-1]
671 module_file = module_file[:-1]
672 for test in tests:
672 for test in tests:
673 if not test.examples:
673 if not test.examples:
674 continue
674 continue
675 if not test.filename:
675 if not test.filename:
676 test.filename = module_file
676 test.filename = module_file
677
677
678 yield DocTestCase(test,
678 yield DocTestCase(test,
679 optionflags=optionflags,
679 optionflags=optionflags,
680 checker=self.checker)
680 checker=self.checker)
681
681
682
682
683 def loadTestsFromFile(self, filename):
683 def loadTestsFromFile(self, filename):
684 #print "ipdoctest - from file", filename # dbg
684 #print "ipdoctest - from file", filename # dbg
685 if is_extension_module(filename):
685 if is_extension_module(filename):
686 for t in self.loadTestsFromExtensionModule(filename):
686 for t in self.loadTestsFromExtensionModule(filename):
687 yield t
687 yield t
688 else:
688 else:
689 if self.extension and anyp(filename.endswith, self.extension):
689 if self.extension and anyp(filename.endswith, self.extension):
690 name = os.path.basename(filename)
690 name = os.path.basename(filename)
691 dh = open(filename)
691 with open(filename) as dh:
692 try:
693 doc = dh.read()
692 doc = dh.read()
694 finally:
695 dh.close()
696 test = self.parser.get_doctest(
693 test = self.parser.get_doctest(
697 doc, globs={'__file__': filename}, name=name,
694 doc, globs={'__file__': filename}, name=name,
698 filename=filename, lineno=0)
695 filename=filename, lineno=0)
699 if test.examples:
696 if test.examples:
700 #print 'FileCase:',test.examples # dbg
697 #print 'FileCase:',test.examples # dbg
701 yield DocFileCase(test)
698 yield DocFileCase(test)
702 else:
699 else:
703 yield False # no tests to load
700 yield False # no tests to load
704
701
705
702
706 class IPythonDoctest(ExtensionDoctest):
703 class IPythonDoctest(ExtensionDoctest):
707 """Nose Plugin that supports doctests in extension modules.
704 """Nose Plugin that supports doctests in extension modules.
708 """
705 """
709 name = 'ipdoctest' # call nosetests with --with-ipdoctest
706 name = 'ipdoctest' # call nosetests with --with-ipdoctest
710 enabled = True
707 enabled = True
711
708
712 def makeTest(self, obj, parent):
709 def makeTest(self, obj, parent):
713 """Look for doctests in the given object, which will be a
710 """Look for doctests in the given object, which will be a
714 function, method or class.
711 function, method or class.
715 """
712 """
716 #print 'Plugin analyzing:', obj, parent # dbg
713 #print 'Plugin analyzing:', obj, parent # dbg
717 # always use whitespace and ellipsis options
714 # always use whitespace and ellipsis options
718 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
715 optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
719
716
720 doctests = self.finder.find(obj, module=getmodule(parent))
717 doctests = self.finder.find(obj, module=getmodule(parent))
721 if doctests:
718 if doctests:
722 for test in doctests:
719 for test in doctests:
723 if len(test.examples) == 0:
720 if len(test.examples) == 0:
724 continue
721 continue
725
722
726 yield DocTestCase(test, obj=obj,
723 yield DocTestCase(test, obj=obj,
727 optionflags=optionflags,
724 optionflags=optionflags,
728 checker=self.checker)
725 checker=self.checker)
729
726
730 def options(self, parser, env=os.environ):
727 def options(self, parser, env=os.environ):
731 #print "Options for nose plugin:", self.name # dbg
728 #print "Options for nose plugin:", self.name # dbg
732 Plugin.options(self, parser, env)
729 Plugin.options(self, parser, env)
733 parser.add_option('--ipdoctest-tests', action='store_true',
730 parser.add_option('--ipdoctest-tests', action='store_true',
734 dest='ipdoctest_tests',
731 dest='ipdoctest_tests',
735 default=env.get('NOSE_IPDOCTEST_TESTS',True),
732 default=env.get('NOSE_IPDOCTEST_TESTS',True),
736 help="Also look for doctests in test modules. "
733 help="Also look for doctests in test modules. "
737 "Note that classes, methods and functions should "
734 "Note that classes, methods and functions should "
738 "have either doctests or non-doctest tests, "
735 "have either doctests or non-doctest tests, "
739 "not both. [NOSE_IPDOCTEST_TESTS]")
736 "not both. [NOSE_IPDOCTEST_TESTS]")
740 parser.add_option('--ipdoctest-extension', action="append",
737 parser.add_option('--ipdoctest-extension', action="append",
741 dest="ipdoctest_extension",
738 dest="ipdoctest_extension",
742 help="Also look for doctests in files with "
739 help="Also look for doctests in files with "
743 "this extension [NOSE_IPDOCTEST_EXTENSION]")
740 "this extension [NOSE_IPDOCTEST_EXTENSION]")
744 # Set the default as a list, if given in env; otherwise
741 # Set the default as a list, if given in env; otherwise
745 # an additional value set on the command line will cause
742 # an additional value set on the command line will cause
746 # an error.
743 # an error.
747 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
744 env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
748 if env_setting is not None:
745 if env_setting is not None:
749 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
746 parser.set_defaults(ipdoctest_extension=tolist(env_setting))
750
747
751 def configure(self, options, config):
748 def configure(self, options, config):
752 #print "Configuring nose plugin:", self.name # dbg
749 #print "Configuring nose plugin:", self.name # dbg
753 Plugin.configure(self, options, config)
750 Plugin.configure(self, options, config)
754 # Pull standard doctest plugin out of config; we will do doctesting
751 # Pull standard doctest plugin out of config; we will do doctesting
755 config.plugins.plugins = [p for p in config.plugins.plugins
752 config.plugins.plugins = [p for p in config.plugins.plugins
756 if p.name != 'doctest']
753 if p.name != 'doctest']
757 self.doctest_tests = options.ipdoctest_tests
754 self.doctest_tests = options.ipdoctest_tests
758 self.extension = tolist(options.ipdoctest_extension)
755 self.extension = tolist(options.ipdoctest_extension)
759
756
760 self.parser = IPDocTestParser()
757 self.parser = IPDocTestParser()
761 self.finder = DocTestFinder(parser=self.parser)
758 self.finder = DocTestFinder(parser=self.parser)
762 self.checker = IPDoctestOutputChecker()
759 self.checker = IPDoctestOutputChecker()
763 self.globs = None
760 self.globs = None
764 self.extraglobs = None
761 self.extraglobs = None
@@ -1,472 +1,471 b''
1 """Generic testing tools.
1 """Generic testing tools.
2
2
3 Authors
3 Authors
4 -------
4 -------
5 - Fernando Perez <Fernando.Perez@berkeley.edu>
5 - Fernando Perez <Fernando.Perez@berkeley.edu>
6 """
6 """
7
7
8
8
9 # Copyright (c) IPython Development Team.
9 # Copyright (c) IPython Development Team.
10 # Distributed under the terms of the Modified BSD License.
10 # Distributed under the terms of the Modified BSD License.
11
11
12 import os
12 import os
13 import re
13 import re
14 import sys
14 import sys
15 import tempfile
15 import tempfile
16
16
17 from contextlib import contextmanager
17 from contextlib import contextmanager
18 from io import StringIO
18 from io import StringIO
19 from subprocess import Popen, PIPE
19 from subprocess import Popen, PIPE
20 from unittest.mock import patch
20 from unittest.mock import patch
21
21
22 try:
22 try:
23 # These tools are used by parts of the runtime, so we make the nose
23 # These tools are used by parts of the runtime, so we make the nose
24 # dependency optional at this point. Nose is a hard dependency to run the
24 # dependency optional at this point. Nose is a hard dependency to run the
25 # test suite, but NOT to use ipython itself.
25 # test suite, but NOT to use ipython itself.
26 import nose.tools as nt
26 import nose.tools as nt
27 has_nose = True
27 has_nose = True
28 except ImportError:
28 except ImportError:
29 has_nose = False
29 has_nose = False
30
30
31 from traitlets.config.loader import Config
31 from traitlets.config.loader import Config
32 from IPython.utils.process import get_output_error_code
32 from IPython.utils.process import get_output_error_code
33 from IPython.utils.text import list_strings
33 from IPython.utils.text import list_strings
34 from IPython.utils.io import temp_pyfile, Tee
34 from IPython.utils.io import temp_pyfile, Tee
35 from IPython.utils import py3compat
35 from IPython.utils import py3compat
36
36
37 from . import decorators as dec
37 from . import decorators as dec
38 from . import skipdoctest
38 from . import skipdoctest
39
39
40
40
41 # The docstring for full_path doctests differently on win32 (different path
41 # The docstring for full_path doctests differently on win32 (different path
42 # separator) so just skip the doctest there. The example remains informative.
42 # separator) so just skip the doctest there. The example remains informative.
43 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
43 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
44
44
45 @doctest_deco
45 @doctest_deco
46 def full_path(startPath,files):
46 def full_path(startPath,files):
47 """Make full paths for all the listed files, based on startPath.
47 """Make full paths for all the listed files, based on startPath.
48
48
49 Only the base part of startPath is kept, since this routine is typically
49 Only the base part of startPath is kept, since this routine is typically
50 used with a script's ``__file__`` variable as startPath. The base of startPath
50 used with a script's ``__file__`` variable as startPath. The base of startPath
51 is then prepended to all the listed files, forming the output list.
51 is then prepended to all the listed files, forming the output list.
52
52
53 Parameters
53 Parameters
54 ----------
54 ----------
55 startPath : string
55 startPath : string
56 Initial path to use as the base for the results. This path is split
56 Initial path to use as the base for the results. This path is split
57 using os.path.split() and only its first component is kept.
57 using os.path.split() and only its first component is kept.
58
58
59 files : string or list
59 files : string or list
60 One or more files.
60 One or more files.
61
61
62 Examples
62 Examples
63 --------
63 --------
64
64
65 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
65 >>> full_path('/foo/bar.py',['a.txt','b.txt'])
66 ['/foo/a.txt', '/foo/b.txt']
66 ['/foo/a.txt', '/foo/b.txt']
67
67
68 >>> full_path('/foo',['a.txt','b.txt'])
68 >>> full_path('/foo',['a.txt','b.txt'])
69 ['/a.txt', '/b.txt']
69 ['/a.txt', '/b.txt']
70
70
71 If a single file is given, the output is still a list::
71 If a single file is given, the output is still a list::
72
72
73 >>> full_path('/foo','a.txt')
73 >>> full_path('/foo','a.txt')
74 ['/a.txt']
74 ['/a.txt']
75 """
75 """
76
76
77 files = list_strings(files)
77 files = list_strings(files)
78 base = os.path.split(startPath)[0]
78 base = os.path.split(startPath)[0]
79 return [ os.path.join(base,f) for f in files ]
79 return [ os.path.join(base,f) for f in files ]
80
80
81
81
82 def parse_test_output(txt):
82 def parse_test_output(txt):
83 """Parse the output of a test run and return errors, failures.
83 """Parse the output of a test run and return errors, failures.
84
84
85 Parameters
85 Parameters
86 ----------
86 ----------
87 txt : str
87 txt : str
88 Text output of a test run, assumed to contain a line of one of the
88 Text output of a test run, assumed to contain a line of one of the
89 following forms::
89 following forms::
90
90
91 'FAILED (errors=1)'
91 'FAILED (errors=1)'
92 'FAILED (failures=1)'
92 'FAILED (failures=1)'
93 'FAILED (errors=1, failures=1)'
93 'FAILED (errors=1, failures=1)'
94
94
95 Returns
95 Returns
96 -------
96 -------
97 nerr, nfail
97 nerr, nfail
98 number of errors and failures.
98 number of errors and failures.
99 """
99 """
100
100
101 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
101 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
102 if err_m:
102 if err_m:
103 nerr = int(err_m.group(1))
103 nerr = int(err_m.group(1))
104 nfail = 0
104 nfail = 0
105 return nerr, nfail
105 return nerr, nfail
106
106
107 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
107 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
108 if fail_m:
108 if fail_m:
109 nerr = 0
109 nerr = 0
110 nfail = int(fail_m.group(1))
110 nfail = int(fail_m.group(1))
111 return nerr, nfail
111 return nerr, nfail
112
112
113 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
113 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
114 re.MULTILINE)
114 re.MULTILINE)
115 if both_m:
115 if both_m:
116 nerr = int(both_m.group(1))
116 nerr = int(both_m.group(1))
117 nfail = int(both_m.group(2))
117 nfail = int(both_m.group(2))
118 return nerr, nfail
118 return nerr, nfail
119
119
120 # If the input didn't match any of these forms, assume no error/failures
120 # If the input didn't match any of these forms, assume no error/failures
121 return 0, 0
121 return 0, 0
122
122
123
123
124 # So nose doesn't think this is a test
124 # So nose doesn't think this is a test
125 parse_test_output.__test__ = False
125 parse_test_output.__test__ = False
126
126
127
127
128 def default_argv():
128 def default_argv():
129 """Return a valid default argv for creating testing instances of ipython"""
129 """Return a valid default argv for creating testing instances of ipython"""
130
130
131 return ['--quick', # so no config file is loaded
131 return ['--quick', # so no config file is loaded
132 # Other defaults to minimize side effects on stdout
132 # Other defaults to minimize side effects on stdout
133 '--colors=NoColor', '--no-term-title','--no-banner',
133 '--colors=NoColor', '--no-term-title','--no-banner',
134 '--autocall=0']
134 '--autocall=0']
135
135
136
136
137 def default_config():
137 def default_config():
138 """Return a config object with good defaults for testing."""
138 """Return a config object with good defaults for testing."""
139 config = Config()
139 config = Config()
140 config.TerminalInteractiveShell.colors = 'NoColor'
140 config.TerminalInteractiveShell.colors = 'NoColor'
141 config.TerminalTerminalInteractiveShell.term_title = False,
141 config.TerminalTerminalInteractiveShell.term_title = False,
142 config.TerminalInteractiveShell.autocall = 0
142 config.TerminalInteractiveShell.autocall = 0
143 f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
143 f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
144 config.HistoryManager.hist_file = f.name
144 config.HistoryManager.hist_file = f.name
145 f.close()
145 f.close()
146 config.HistoryManager.db_cache_size = 10000
146 config.HistoryManager.db_cache_size = 10000
147 return config
147 return config
148
148
149
149
150 def get_ipython_cmd(as_string=False):
150 def get_ipython_cmd(as_string=False):
151 """
151 """
152 Return appropriate IPython command line name. By default, this will return
152 Return appropriate IPython command line name. By default, this will return
153 a list that can be used with subprocess.Popen, for example, but passing
153 a list that can be used with subprocess.Popen, for example, but passing
154 `as_string=True` allows for returning the IPython command as a string.
154 `as_string=True` allows for returning the IPython command as a string.
155
155
156 Parameters
156 Parameters
157 ----------
157 ----------
158 as_string: bool
158 as_string: bool
159 Flag to allow to return the command as a string.
159 Flag to allow to return the command as a string.
160 """
160 """
161 ipython_cmd = [sys.executable, "-m", "IPython"]
161 ipython_cmd = [sys.executable, "-m", "IPython"]
162
162
163 if as_string:
163 if as_string:
164 ipython_cmd = " ".join(ipython_cmd)
164 ipython_cmd = " ".join(ipython_cmd)
165
165
166 return ipython_cmd
166 return ipython_cmd
167
167
168 def ipexec(fname, options=None, commands=()):
168 def ipexec(fname, options=None, commands=()):
169 """Utility to call 'ipython filename'.
169 """Utility to call 'ipython filename'.
170
170
171 Starts IPython with a minimal and safe configuration to make startup as fast
171 Starts IPython with a minimal and safe configuration to make startup as fast
172 as possible.
172 as possible.
173
173
174 Note that this starts IPython in a subprocess!
174 Note that this starts IPython in a subprocess!
175
175
176 Parameters
176 Parameters
177 ----------
177 ----------
178 fname : str
178 fname : str
179 Name of file to be executed (should have .py or .ipy extension).
179 Name of file to be executed (should have .py or .ipy extension).
180
180
181 options : optional, list
181 options : optional, list
182 Extra command-line flags to be passed to IPython.
182 Extra command-line flags to be passed to IPython.
183
183
184 commands : optional, list
184 commands : optional, list
185 Commands to send in on stdin
185 Commands to send in on stdin
186
186
187 Returns
187 Returns
188 -------
188 -------
189 ``(stdout, stderr)`` of ipython subprocess.
189 ``(stdout, stderr)`` of ipython subprocess.
190 """
190 """
191 if options is None: options = []
191 if options is None: options = []
192
192
193 cmdargs = default_argv() + options
193 cmdargs = default_argv() + options
194
194
195 test_dir = os.path.dirname(__file__)
195 test_dir = os.path.dirname(__file__)
196
196
197 ipython_cmd = get_ipython_cmd()
197 ipython_cmd = get_ipython_cmd()
198 # Absolute path for filename
198 # Absolute path for filename
199 full_fname = os.path.join(test_dir, fname)
199 full_fname = os.path.join(test_dir, fname)
200 full_cmd = ipython_cmd + cmdargs + [full_fname]
200 full_cmd = ipython_cmd + cmdargs + [full_fname]
201 env = os.environ.copy()
201 env = os.environ.copy()
202 # FIXME: ignore all warnings in ipexec while we have shims
202 # FIXME: ignore all warnings in ipexec while we have shims
203 # should we keep suppressing warnings here, even after removing shims?
203 # should we keep suppressing warnings here, even after removing shims?
204 env['PYTHONWARNINGS'] = 'ignore'
204 env['PYTHONWARNINGS'] = 'ignore'
205 # env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
205 # env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr
206 for k, v in env.items():
206 for k, v in env.items():
207 # Debug a bizarre failure we've seen on Windows:
207 # Debug a bizarre failure we've seen on Windows:
208 # TypeError: environment can only contain strings
208 # TypeError: environment can only contain strings
209 if not isinstance(v, str):
209 if not isinstance(v, str):
210 print(k, v)
210 print(k, v)
211 p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
211 p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
212 out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None)
212 out, err = p.communicate(input=py3compat.encode('\n'.join(commands)) or None)
213 out, err = py3compat.decode(out), py3compat.decode(err)
213 out, err = py3compat.decode(out), py3compat.decode(err)
214 # `import readline` causes 'ESC[?1034h' to be output sometimes,
214 # `import readline` causes 'ESC[?1034h' to be output sometimes,
215 # so strip that out before doing comparisons
215 # so strip that out before doing comparisons
216 if out:
216 if out:
217 out = re.sub(r'\x1b\[[^h]+h', '', out)
217 out = re.sub(r'\x1b\[[^h]+h', '', out)
218 return out, err
218 return out, err
219
219
220
220
221 def ipexec_validate(fname, expected_out, expected_err='',
221 def ipexec_validate(fname, expected_out, expected_err='',
222 options=None, commands=()):
222 options=None, commands=()):
223 """Utility to call 'ipython filename' and validate output/error.
223 """Utility to call 'ipython filename' and validate output/error.
224
224
225 This function raises an AssertionError if the validation fails.
225 This function raises an AssertionError if the validation fails.
226
226
227 Note that this starts IPython in a subprocess!
227 Note that this starts IPython in a subprocess!
228
228
229 Parameters
229 Parameters
230 ----------
230 ----------
231 fname : str
231 fname : str
232 Name of the file to be executed (should have .py or .ipy extension).
232 Name of the file to be executed (should have .py or .ipy extension).
233
233
234 expected_out : str
234 expected_out : str
235 Expected stdout of the process.
235 Expected stdout of the process.
236
236
237 expected_err : optional, str
237 expected_err : optional, str
238 Expected stderr of the process.
238 Expected stderr of the process.
239
239
240 options : optional, list
240 options : optional, list
241 Extra command-line flags to be passed to IPython.
241 Extra command-line flags to be passed to IPython.
242
242
243 Returns
243 Returns
244 -------
244 -------
245 None
245 None
246 """
246 """
247
247
248 import nose.tools as nt
248 import nose.tools as nt
249
249
250 out, err = ipexec(fname, options, commands)
250 out, err = ipexec(fname, options, commands)
251 #print 'OUT', out # dbg
251 #print 'OUT', out # dbg
252 #print 'ERR', err # dbg
252 #print 'ERR', err # dbg
253 # If there are any errors, we must check those before stdout, as they may be
253 # If there are any errors, we must check those before stdout, as they may be
254 # more informative than simply having an empty stdout.
254 # more informative than simply having an empty stdout.
255 if err:
255 if err:
256 if expected_err:
256 if expected_err:
257 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
257 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
258 else:
258 else:
259 raise ValueError('Running file %r produced error: %r' %
259 raise ValueError('Running file %r produced error: %r' %
260 (fname, err))
260 (fname, err))
261 # If no errors or output on stderr was expected, match stdout
261 # If no errors or output on stderr was expected, match stdout
262 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
262 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
263
263
264
264
265 class TempFileMixin(object):
265 class TempFileMixin(object):
266 """Utility class to create temporary Python/IPython files.
266 """Utility class to create temporary Python/IPython files.
267
267
268 Meant as a mixin class for test cases."""
268 Meant as a mixin class for test cases."""
269
269
270 def mktmp(self, src, ext='.py'):
270 def mktmp(self, src, ext='.py'):
271 """Make a valid python temp file."""
271 """Make a valid python temp file."""
272 fname, f = temp_pyfile(src, ext)
272 fname, f = temp_pyfile(src, ext)
273 if not hasattr(self, 'tmps'):
273 if not hasattr(self, 'tmps'):
274 self.tmps=[]
274 self.tmps=[]
275 self.tmps.append((f, fname))
275 self.tmps.append((f, fname))
276 self.fname = fname
276 self.fname = fname
277
277
278 def tearDown(self):
278 def tearDown(self):
279 # If the tmpfile wasn't made because of skipped tests, like in
279 # If the tmpfile wasn't made because of skipped tests, like in
280 # win32, there's nothing to cleanup.
280 # win32, there's nothing to cleanup.
281 if hasattr(self, 'tmps'):
281 if hasattr(self, 'tmps'):
282 for f,fname in self.tmps:
282 for f,fname in self.tmps:
283 # If the tmpfile wasn't made because of skipped tests, like in
283 # If the tmpfile wasn't made because of skipped tests, like in
284 # win32, there's nothing to cleanup.
284 # win32, there's nothing to cleanup.
285 f.close()
285 f.close()
286 try:
286 try:
287 os.unlink(fname)
287 os.unlink(fname)
288 except:
288 except:
289 # On Windows, even though we close the file, we still can't
289 # On Windows, even though we close the file, we still can't
290 # delete it. I have no clue why
290 # delete it. I have no clue why
291 pass
291 pass
292
292
293 def __enter__(self):
293 def __enter__(self):
294 return self
294 return self
295
295
296 def __exit__(self, exc_type, exc_value, traceback):
296 def __exit__(self, exc_type, exc_value, traceback):
297 self.tearDown()
297 self.tearDown()
298
298
299
299
300 pair_fail_msg = ("Testing {0}\n\n"
300 pair_fail_msg = ("Testing {0}\n\n"
301 "In:\n"
301 "In:\n"
302 " {1!r}\n"
302 " {1!r}\n"
303 "Expected:\n"
303 "Expected:\n"
304 " {2!r}\n"
304 " {2!r}\n"
305 "Got:\n"
305 "Got:\n"
306 " {3!r}\n")
306 " {3!r}\n")
307 def check_pairs(func, pairs):
307 def check_pairs(func, pairs):
308 """Utility function for the common case of checking a function with a
308 """Utility function for the common case of checking a function with a
309 sequence of input/output pairs.
309 sequence of input/output pairs.
310
310
311 Parameters
311 Parameters
312 ----------
312 ----------
313 func : callable
313 func : callable
314 The function to be tested. Should accept a single argument.
314 The function to be tested. Should accept a single argument.
315 pairs : iterable
315 pairs : iterable
316 A list of (input, expected_output) tuples.
316 A list of (input, expected_output) tuples.
317
317
318 Returns
318 Returns
319 -------
319 -------
320 None. Raises an AssertionError if any output does not match the expected
320 None. Raises an AssertionError if any output does not match the expected
321 value.
321 value.
322 """
322 """
323 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
323 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
324 for inp, expected in pairs:
324 for inp, expected in pairs:
325 out = func(inp)
325 out = func(inp)
326 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
326 assert out == expected, pair_fail_msg.format(name, inp, expected, out)
327
327
328
328
329 MyStringIO = StringIO
329 MyStringIO = StringIO
330
330
331 _re_type = type(re.compile(r''))
331 _re_type = type(re.compile(r''))
332
332
333 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
333 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
334 -------
334 -------
335 {2!s}
335 {2!s}
336 -------
336 -------
337 """
337 """
338
338
339 class AssertPrints(object):
339 class AssertPrints(object):
340 """Context manager for testing that code prints certain text.
340 """Context manager for testing that code prints certain text.
341
341
342 Examples
342 Examples
343 --------
343 --------
344 >>> with AssertPrints("abc", suppress=False):
344 >>> with AssertPrints("abc", suppress=False):
345 ... print("abcd")
345 ... print("abcd")
346 ... print("def")
346 ... print("def")
347 ...
347 ...
348 abcd
348 abcd
349 def
349 def
350 """
350 """
351 def __init__(self, s, channel='stdout', suppress=True):
351 def __init__(self, s, channel='stdout', suppress=True):
352 self.s = s
352 self.s = s
353 if isinstance(self.s, (str, _re_type)):
353 if isinstance(self.s, (str, _re_type)):
354 self.s = [self.s]
354 self.s = [self.s]
355 self.channel = channel
355 self.channel = channel
356 self.suppress = suppress
356 self.suppress = suppress
357
357
358 def __enter__(self):
358 def __enter__(self):
359 self.orig_stream = getattr(sys, self.channel)
359 self.orig_stream = getattr(sys, self.channel)
360 self.buffer = MyStringIO()
360 self.buffer = MyStringIO()
361 self.tee = Tee(self.buffer, channel=self.channel)
361 self.tee = Tee(self.buffer, channel=self.channel)
362 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
362 setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
363
363
364 def __exit__(self, etype, value, traceback):
364 def __exit__(self, etype, value, traceback):
365 try:
365 try:
366 if value is not None:
366 if value is not None:
367 # If an error was raised, don't check anything else
367 # If an error was raised, don't check anything else
368 return False
368 return False
369 self.tee.flush()
369 self.tee.flush()
370 setattr(sys, self.channel, self.orig_stream)
370 setattr(sys, self.channel, self.orig_stream)
371 printed = self.buffer.getvalue()
371 printed = self.buffer.getvalue()
372 for s in self.s:
372 for s in self.s:
373 if isinstance(s, _re_type):
373 if isinstance(s, _re_type):
374 assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
374 assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
375 else:
375 else:
376 assert s in printed, notprinted_msg.format(s, self.channel, printed)
376 assert s in printed, notprinted_msg.format(s, self.channel, printed)
377 return False
377 return False
378 finally:
378 finally:
379 self.tee.close()
379 self.tee.close()
380
380
381 printed_msg = """Found {0!r} in printed output (on {1}):
381 printed_msg = """Found {0!r} in printed output (on {1}):
382 -------
382 -------
383 {2!s}
383 {2!s}
384 -------
384 -------
385 """
385 """
386
386
387 class AssertNotPrints(AssertPrints):
387 class AssertNotPrints(AssertPrints):
388 """Context manager for checking that certain output *isn't* produced.
388 """Context manager for checking that certain output *isn't* produced.
389
389
390 Counterpart of AssertPrints"""
390 Counterpart of AssertPrints"""
391 def __exit__(self, etype, value, traceback):
391 def __exit__(self, etype, value, traceback):
392 try:
392 try:
393 if value is not None:
393 if value is not None:
394 # If an error was raised, don't check anything else
394 # If an error was raised, don't check anything else
395 self.tee.close()
395 self.tee.close()
396 return False
396 return False
397 self.tee.flush()
397 self.tee.flush()
398 setattr(sys, self.channel, self.orig_stream)
398 setattr(sys, self.channel, self.orig_stream)
399 printed = self.buffer.getvalue()
399 printed = self.buffer.getvalue()
400 for s in self.s:
400 for s in self.s:
401 if isinstance(s, _re_type):
401 if isinstance(s, _re_type):
402 assert not s.search(printed),printed_msg.format(
402 assert not s.search(printed),printed_msg.format(
403 s.pattern, self.channel, printed)
403 s.pattern, self.channel, printed)
404 else:
404 else:
405 assert s not in printed, printed_msg.format(
405 assert s not in printed, printed_msg.format(
406 s, self.channel, printed)
406 s, self.channel, printed)
407 return False
407 return False
408 finally:
408 finally:
409 self.tee.close()
409 self.tee.close()
410
410
411 @contextmanager
411 @contextmanager
412 def mute_warn():
412 def mute_warn():
413 from IPython.utils import warn
413 from IPython.utils import warn
414 save_warn = warn.warn
414 save_warn = warn.warn
415 warn.warn = lambda *a, **kw: None
415 warn.warn = lambda *a, **kw: None
416 try:
416 try:
417 yield
417 yield
418 finally:
418 finally:
419 warn.warn = save_warn
419 warn.warn = save_warn
420
420
421 @contextmanager
421 @contextmanager
422 def make_tempfile(name):
422 def make_tempfile(name):
423 """ Create an empty, named, temporary file for the duration of the context.
423 """ Create an empty, named, temporary file for the duration of the context.
424 """
424 """
425 f = open(name, 'w')
425 open(name, 'w').close()
426 f.close()
427 try:
426 try:
428 yield
427 yield
429 finally:
428 finally:
430 os.unlink(name)
429 os.unlink(name)
431
430
432 def fake_input(inputs):
431 def fake_input(inputs):
433 """Temporarily replace the input() function to return the given values
432 """Temporarily replace the input() function to return the given values
434
433
435 Use as a context manager:
434 Use as a context manager:
436
435
437 with fake_input(['result1', 'result2']):
436 with fake_input(['result1', 'result2']):
438 ...
437 ...
439
438
440 Values are returned in order. If input() is called again after the last value
439 Values are returned in order. If input() is called again after the last value
441 was used, EOFError is raised.
440 was used, EOFError is raised.
442 """
441 """
443 it = iter(inputs)
442 it = iter(inputs)
444 def mock_input(prompt=''):
443 def mock_input(prompt=''):
445 try:
444 try:
446 return next(it)
445 return next(it)
447 except StopIteration:
446 except StopIteration:
448 raise EOFError('No more inputs given')
447 raise EOFError('No more inputs given')
449
448
450 return patch('builtins.input', mock_input)
449 return patch('builtins.input', mock_input)
451
450
452 def help_output_test(subcommand=''):
451 def help_output_test(subcommand=''):
453 """test that `ipython [subcommand] -h` works"""
452 """test that `ipython [subcommand] -h` works"""
454 cmd = get_ipython_cmd() + [subcommand, '-h']
453 cmd = get_ipython_cmd() + [subcommand, '-h']
455 out, err, rc = get_output_error_code(cmd)
454 out, err, rc = get_output_error_code(cmd)
456 nt.assert_equal(rc, 0, err)
455 nt.assert_equal(rc, 0, err)
457 nt.assert_not_in("Traceback", err)
456 nt.assert_not_in("Traceback", err)
458 nt.assert_in("Options", out)
457 nt.assert_in("Options", out)
459 nt.assert_in("--help-all", out)
458 nt.assert_in("--help-all", out)
460 return out, err
459 return out, err
461
460
462
461
463 def help_all_output_test(subcommand=''):
462 def help_all_output_test(subcommand=''):
464 """test that `ipython [subcommand] --help-all` works"""
463 """test that `ipython [subcommand] --help-all` works"""
465 cmd = get_ipython_cmd() + [subcommand, '--help-all']
464 cmd = get_ipython_cmd() + [subcommand, '--help-all']
466 out, err, rc = get_output_error_code(cmd)
465 out, err, rc = get_output_error_code(cmd)
467 nt.assert_equal(rc, 0, err)
466 nt.assert_equal(rc, 0, err)
468 nt.assert_not_in("Traceback", err)
467 nt.assert_not_in("Traceback", err)
469 nt.assert_in("Options", out)
468 nt.assert_in("Options", out)
470 nt.assert_in("Class", out)
469 nt.assert_in("Class", out)
471 return out, err
470 return out, err
472
471
@@ -1,115 +1,114 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """Tests for IPython.utils.module_paths.py"""
2 """Tests for IPython.utils.module_paths.py"""
3
3
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2008-2011 The IPython Development Team
5 # Copyright (C) 2008-2011 The IPython Development Team
6 #
6 #
7 # Distributed under the terms of the BSD License. The full license is in
7 # Distributed under the terms of the BSD License. The full license is in
8 # the file COPYING, distributed as part of this software.
8 # the file COPYING, distributed as part of this software.
9 #-----------------------------------------------------------------------------
9 #-----------------------------------------------------------------------------
10
10
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12 # Imports
12 # Imports
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14
14
15
15
16 import os
16 import os
17 import shutil
17 import shutil
18 import sys
18 import sys
19 import tempfile
19 import tempfile
20
20
21 from os.path import join, abspath, split
21 from os.path import join, abspath, split
22
22
23 from IPython.testing.tools import make_tempfile
23 from IPython.testing.tools import make_tempfile
24
24
25 import IPython.utils.module_paths as mp
25 import IPython.utils.module_paths as mp
26
26
27 import nose.tools as nt
27 import nose.tools as nt
28
28
29 env = os.environ
29 env = os.environ
30 TEST_FILE_PATH = split(abspath(__file__))[0]
30 TEST_FILE_PATH = split(abspath(__file__))[0]
31
31
32 TMP_TEST_DIR = tempfile.mkdtemp(suffix='with.dot')
32 TMP_TEST_DIR = tempfile.mkdtemp(suffix='with.dot')
33 #
33 #
34 # Setup/teardown functions/decorators
34 # Setup/teardown functions/decorators
35 #
35 #
36
36
37 old_syspath = sys.path
37 old_syspath = sys.path
38
38
39 def make_empty_file(fname):
39 def make_empty_file(fname):
40 f = open(fname, 'w')
40 open(fname, 'w').close()
41 f.close()
42
41
43
42
44 def setup():
43 def setup():
45 """Setup testenvironment for the module:
44 """Setup testenvironment for the module:
46
45
47 """
46 """
48 # Do not mask exceptions here. In particular, catching WindowsError is a
47 # Do not mask exceptions here. In particular, catching WindowsError is a
49 # problem because that exception is only defined on Windows...
48 # problem because that exception is only defined on Windows...
50 os.makedirs(join(TMP_TEST_DIR, "xmod"))
49 os.makedirs(join(TMP_TEST_DIR, "xmod"))
51 os.makedirs(join(TMP_TEST_DIR, "nomod"))
50 os.makedirs(join(TMP_TEST_DIR, "nomod"))
52 make_empty_file(join(TMP_TEST_DIR, "xmod/__init__.py"))
51 make_empty_file(join(TMP_TEST_DIR, "xmod/__init__.py"))
53 make_empty_file(join(TMP_TEST_DIR, "xmod/sub.py"))
52 make_empty_file(join(TMP_TEST_DIR, "xmod/sub.py"))
54 make_empty_file(join(TMP_TEST_DIR, "pack.py"))
53 make_empty_file(join(TMP_TEST_DIR, "pack.py"))
55 make_empty_file(join(TMP_TEST_DIR, "packpyc.pyc"))
54 make_empty_file(join(TMP_TEST_DIR, "packpyc.pyc"))
56 sys.path = [TMP_TEST_DIR]
55 sys.path = [TMP_TEST_DIR]
57
56
58 def teardown():
57 def teardown():
59 """Teardown testenvironment for the module:
58 """Teardown testenvironment for the module:
60
59
61 - Remove tempdir
60 - Remove tempdir
62 - restore sys.path
61 - restore sys.path
63 """
62 """
64 # Note: we remove the parent test dir, which is the root of all test
63 # Note: we remove the parent test dir, which is the root of all test
65 # subdirs we may have created. Use shutil instead of os.removedirs, so
64 # subdirs we may have created. Use shutil instead of os.removedirs, so
66 # that non-empty directories are all recursively removed.
65 # that non-empty directories are all recursively removed.
67 shutil.rmtree(TMP_TEST_DIR)
66 shutil.rmtree(TMP_TEST_DIR)
68 sys.path = old_syspath
67 sys.path = old_syspath
69
68
70 def test_tempdir():
69 def test_tempdir():
71 """
70 """
72 Ensure the test are done with a temporary file that have a dot somewhere.
71 Ensure the test are done with a temporary file that have a dot somewhere.
73 """
72 """
74 nt.assert_in('.',TMP_TEST_DIR)
73 nt.assert_in('.',TMP_TEST_DIR)
75
74
76
75
77 def test_find_mod_1():
76 def test_find_mod_1():
78 """
77 """
79 Search for a directory's file path.
78 Search for a directory's file path.
80 Expected output: a path to that directory's __init__.py file.
79 Expected output: a path to that directory's __init__.py file.
81 """
80 """
82 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
81 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
83 nt.assert_equal(mp.find_mod("xmod"), modpath)
82 nt.assert_equal(mp.find_mod("xmod"), modpath)
84
83
85 def test_find_mod_2():
84 def test_find_mod_2():
86 """
85 """
87 Search for a directory's file path.
86 Search for a directory's file path.
88 Expected output: a path to that directory's __init__.py file.
87 Expected output: a path to that directory's __init__.py file.
89 TODO: Confirm why this is a duplicate test.
88 TODO: Confirm why this is a duplicate test.
90 """
89 """
91 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
90 modpath = join(TMP_TEST_DIR, "xmod", "__init__.py")
92 nt.assert_equal(mp.find_mod("xmod"), modpath)
91 nt.assert_equal(mp.find_mod("xmod"), modpath)
93
92
94 def test_find_mod_3():
93 def test_find_mod_3():
95 """
94 """
96 Search for a directory + a filename without its .py extension
95 Search for a directory + a filename without its .py extension
97 Expected output: full path with .py extension.
96 Expected output: full path with .py extension.
98 """
97 """
99 modpath = join(TMP_TEST_DIR, "xmod", "sub.py")
98 modpath = join(TMP_TEST_DIR, "xmod", "sub.py")
100 nt.assert_equal(mp.find_mod("xmod.sub"), modpath)
99 nt.assert_equal(mp.find_mod("xmod.sub"), modpath)
101
100
102 def test_find_mod_4():
101 def test_find_mod_4():
103 """
102 """
104 Search for a filename without its .py extension
103 Search for a filename without its .py extension
105 Expected output: full path with .py extension
104 Expected output: full path with .py extension
106 """
105 """
107 modpath = join(TMP_TEST_DIR, "pack.py")
106 modpath = join(TMP_TEST_DIR, "pack.py")
108 nt.assert_equal(mp.find_mod("pack"), modpath)
107 nt.assert_equal(mp.find_mod("pack"), modpath)
109
108
110 def test_find_mod_5():
109 def test_find_mod_5():
111 """
110 """
112 Search for a filename with a .pyc extension
111 Search for a filename with a .pyc extension
113 Expected output: TODO: do we exclude or include .pyc files?
112 Expected output: TODO: do we exclude or include .pyc files?
114 """
113 """
115 nt.assert_equal(mp.find_mod("packpyc"), None)
114 nt.assert_equal(mp.find_mod("packpyc"), None)
@@ -1,31 +1,31 b''
1 import io
1 import io
2 import os.path
2 import os.path
3 import nose.tools as nt
3 import nose.tools as nt
4
4
5 from IPython.utils import openpy
5 from IPython.utils import openpy
6
6
7 mydir = os.path.dirname(__file__)
7 mydir = os.path.dirname(__file__)
8 nonascii_path = os.path.join(mydir, '../../core/tests/nonascii.py')
8 nonascii_path = os.path.join(mydir, '../../core/tests/nonascii.py')
9
9
10 def test_detect_encoding():
10 def test_detect_encoding():
11 f = open(nonascii_path, 'rb')
11 with open(nonascii_path, 'rb') as f:
12 enc, lines = openpy.detect_encoding(f.readline)
12 enc, lines = openpy.detect_encoding(f.readline)
13 nt.assert_equal(enc, 'iso-8859-5')
13 nt.assert_equal(enc, 'iso-8859-5')
14
14
15 def test_read_file():
15 def test_read_file():
16 read_specified_enc = io.open(nonascii_path, encoding='iso-8859-5').read()
16 read_specified_enc = io.open(nonascii_path, encoding='iso-8859-5').read()
17 read_detected_enc = openpy.read_py_file(nonascii_path, skip_encoding_cookie=False)
17 read_detected_enc = openpy.read_py_file(nonascii_path, skip_encoding_cookie=False)
18 nt.assert_equal(read_detected_enc, read_specified_enc)
18 nt.assert_equal(read_detected_enc, read_specified_enc)
19 assert u'coding: iso-8859-5' in read_detected_enc
19 assert u'coding: iso-8859-5' in read_detected_enc
20
20
21 read_strip_enc_cookie = openpy.read_py_file(nonascii_path, skip_encoding_cookie=True)
21 read_strip_enc_cookie = openpy.read_py_file(nonascii_path, skip_encoding_cookie=True)
22 assert u'coding: iso-8859-5' not in read_strip_enc_cookie
22 assert u'coding: iso-8859-5' not in read_strip_enc_cookie
23
23
24 def test_source_to_unicode():
24 def test_source_to_unicode():
25 with io.open(nonascii_path, 'rb') as f:
25 with io.open(nonascii_path, 'rb') as f:
26 source_bytes = f.read()
26 source_bytes = f.read()
27 nt.assert_equal(openpy.source_to_unicode(source_bytes, skip_encoding_cookie=False).splitlines(),
27 nt.assert_equal(openpy.source_to_unicode(source_bytes, skip_encoding_cookie=False).splitlines(),
28 source_bytes.decode('iso-8859-5').splitlines())
28 source_bytes.decode('iso-8859-5').splitlines())
29
29
30 source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
30 source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
31 nt.assert_not_in(u'coding: iso-8859-5', source_no_cookie)
31 nt.assert_not_in(u'coding: iso-8859-5', source_no_cookie)
@@ -1,455 +1,453 b''
1 """Attempt to generate templates for module reference with Sphinx
1 """Attempt to generate templates for module reference with Sphinx
2
2
3 XXX - we exclude extension modules
3 XXX - we exclude extension modules
4
4
5 To include extension modules, first identify them as valid in the
5 To include extension modules, first identify them as valid in the
6 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
6 ``_uri2path`` method, then handle them in the ``_parse_module`` script.
7
7
8 We get functions and classes by parsing the text of .py files.
8 We get functions and classes by parsing the text of .py files.
9 Alternatively we could import the modules for discovery, and we'd have
9 Alternatively we could import the modules for discovery, and we'd have
10 to do that for extension modules. This would involve changing the
10 to do that for extension modules. This would involve changing the
11 ``_parse_module`` method to work via import and introspection, and
11 ``_parse_module`` method to work via import and introspection, and
12 might involve changing ``discover_modules`` (which determines which
12 might involve changing ``discover_modules`` (which determines which
13 files are modules, and therefore which module URIs will be passed to
13 files are modules, and therefore which module URIs will be passed to
14 ``_parse_module``).
14 ``_parse_module``).
15
15
16 NOTE: this is a modified version of a script originally shipped with the
16 NOTE: this is a modified version of a script originally shipped with the
17 PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
17 PyMVPA project, which we've adapted for NIPY use. PyMVPA is an MIT-licensed
18 project."""
18 project."""
19
19
20
20
21 # Stdlib imports
21 # Stdlib imports
22 import ast
22 import ast
23 import inspect
23 import inspect
24 import os
24 import os
25 import re
25 import re
26 from importlib import import_module
26 from importlib import import_module
27
27
28
28
29 class Obj(object):
29 class Obj(object):
30 '''Namespace to hold arbitrary information.'''
30 '''Namespace to hold arbitrary information.'''
31 def __init__(self, **kwargs):
31 def __init__(self, **kwargs):
32 for k, v in kwargs.items():
32 for k, v in kwargs.items():
33 setattr(self, k, v)
33 setattr(self, k, v)
34
34
35 class FuncClsScanner(ast.NodeVisitor):
35 class FuncClsScanner(ast.NodeVisitor):
36 """Scan a module for top-level functions and classes.
36 """Scan a module for top-level functions and classes.
37
37
38 Skips objects with an @undoc decorator, or a name starting with '_'.
38 Skips objects with an @undoc decorator, or a name starting with '_'.
39 """
39 """
40 def __init__(self):
40 def __init__(self):
41 ast.NodeVisitor.__init__(self)
41 ast.NodeVisitor.__init__(self)
42 self.classes = []
42 self.classes = []
43 self.classes_seen = set()
43 self.classes_seen = set()
44 self.functions = []
44 self.functions = []
45
45
46 @staticmethod
46 @staticmethod
47 def has_undoc_decorator(node):
47 def has_undoc_decorator(node):
48 return any(isinstance(d, ast.Name) and d.id == 'undoc' \
48 return any(isinstance(d, ast.Name) and d.id == 'undoc' \
49 for d in node.decorator_list)
49 for d in node.decorator_list)
50
50
51 def visit_If(self, node):
51 def visit_If(self, node):
52 if isinstance(node.test, ast.Compare) \
52 if isinstance(node.test, ast.Compare) \
53 and isinstance(node.test.left, ast.Name) \
53 and isinstance(node.test.left, ast.Name) \
54 and node.test.left.id == '__name__':
54 and node.test.left.id == '__name__':
55 return # Ignore classes defined in "if __name__ == '__main__':"
55 return # Ignore classes defined in "if __name__ == '__main__':"
56
56
57 self.generic_visit(node)
57 self.generic_visit(node)
58
58
59 def visit_FunctionDef(self, node):
59 def visit_FunctionDef(self, node):
60 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
60 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
61 and node.name not in self.functions:
61 and node.name not in self.functions:
62 self.functions.append(node.name)
62 self.functions.append(node.name)
63
63
64 def visit_ClassDef(self, node):
64 def visit_ClassDef(self, node):
65 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
65 if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
66 and node.name not in self.classes_seen:
66 and node.name not in self.classes_seen:
67 cls = Obj(name=node.name)
67 cls = Obj(name=node.name)
68 cls.has_init = any(isinstance(n, ast.FunctionDef) and \
68 cls.has_init = any(isinstance(n, ast.FunctionDef) and \
69 n.name=='__init__' for n in node.body)
69 n.name=='__init__' for n in node.body)
70 self.classes.append(cls)
70 self.classes.append(cls)
71 self.classes_seen.add(node.name)
71 self.classes_seen.add(node.name)
72
72
73 def scan(self, mod):
73 def scan(self, mod):
74 self.visit(mod)
74 self.visit(mod)
75 return self.functions, self.classes
75 return self.functions, self.classes
76
76
77 # Functions and classes
77 # Functions and classes
78 class ApiDocWriter(object):
78 class ApiDocWriter(object):
79 ''' Class for automatic detection and parsing of API docs
79 ''' Class for automatic detection and parsing of API docs
80 to Sphinx-parsable reST format'''
80 to Sphinx-parsable reST format'''
81
81
82 # only separating first two levels
82 # only separating first two levels
83 rst_section_levels = ['*', '=', '-', '~', '^']
83 rst_section_levels = ['*', '=', '-', '~', '^']
84
84
85 def __init__(self,
85 def __init__(self,
86 package_name,
86 package_name,
87 rst_extension='.rst',
87 rst_extension='.rst',
88 package_skip_patterns=None,
88 package_skip_patterns=None,
89 module_skip_patterns=None,
89 module_skip_patterns=None,
90 names_from__all__=None,
90 names_from__all__=None,
91 ):
91 ):
92 ''' Initialize package for parsing
92 ''' Initialize package for parsing
93
93
94 Parameters
94 Parameters
95 ----------
95 ----------
96 package_name : string
96 package_name : string
97 Name of the top-level package. *package_name* must be the
97 Name of the top-level package. *package_name* must be the
98 name of an importable package
98 name of an importable package
99 rst_extension : string, optional
99 rst_extension : string, optional
100 Extension for reST files, default '.rst'
100 Extension for reST files, default '.rst'
101 package_skip_patterns : None or sequence of {strings, regexps}
101 package_skip_patterns : None or sequence of {strings, regexps}
102 Sequence of strings giving URIs of packages to be excluded
102 Sequence of strings giving URIs of packages to be excluded
103 Operates on the package path, starting at (including) the
103 Operates on the package path, starting at (including) the
104 first dot in the package path, after *package_name* - so,
104 first dot in the package path, after *package_name* - so,
105 if *package_name* is ``sphinx``, then ``sphinx.util`` will
105 if *package_name* is ``sphinx``, then ``sphinx.util`` will
106 result in ``.util`` being passed for earching by these
106 result in ``.util`` being passed for earching by these
107 regexps. If is None, gives default. Default is:
107 regexps. If is None, gives default. Default is:
108 ['\.tests$']
108 ['\.tests$']
109 module_skip_patterns : None or sequence
109 module_skip_patterns : None or sequence
110 Sequence of strings giving URIs of modules to be excluded
110 Sequence of strings giving URIs of modules to be excluded
111 Operates on the module name including preceding URI path,
111 Operates on the module name including preceding URI path,
112 back to the first dot after *package_name*. For example
112 back to the first dot after *package_name*. For example
113 ``sphinx.util.console`` results in the string to search of
113 ``sphinx.util.console`` results in the string to search of
114 ``.util.console``
114 ``.util.console``
115 If is None, gives default. Default is:
115 If is None, gives default. Default is:
116 ['\.setup$', '\._']
116 ['\.setup$', '\._']
117 names_from__all__ : set, optional
117 names_from__all__ : set, optional
118 Modules listed in here will be scanned by doing ``from mod import *``,
118 Modules listed in here will be scanned by doing ``from mod import *``,
119 rather than finding function and class definitions by scanning the
119 rather than finding function and class definitions by scanning the
120 AST. This is intended for API modules which expose things defined in
120 AST. This is intended for API modules which expose things defined in
121 other files. Modules listed here must define ``__all__`` to avoid
121 other files. Modules listed here must define ``__all__`` to avoid
122 exposing everything they import.
122 exposing everything they import.
123 '''
123 '''
124 if package_skip_patterns is None:
124 if package_skip_patterns is None:
125 package_skip_patterns = ['\\.tests$']
125 package_skip_patterns = ['\\.tests$']
126 if module_skip_patterns is None:
126 if module_skip_patterns is None:
127 module_skip_patterns = ['\\.setup$', '\\._']
127 module_skip_patterns = ['\\.setup$', '\\._']
128 self.package_name = package_name
128 self.package_name = package_name
129 self.rst_extension = rst_extension
129 self.rst_extension = rst_extension
130 self.package_skip_patterns = package_skip_patterns
130 self.package_skip_patterns = package_skip_patterns
131 self.module_skip_patterns = module_skip_patterns
131 self.module_skip_patterns = module_skip_patterns
132 self.names_from__all__ = names_from__all__ or set()
132 self.names_from__all__ = names_from__all__ or set()
133
133
134 def get_package_name(self):
134 def get_package_name(self):
135 return self._package_name
135 return self._package_name
136
136
137 def set_package_name(self, package_name):
137 def set_package_name(self, package_name):
138 ''' Set package_name
138 ''' Set package_name
139
139
140 >>> docwriter = ApiDocWriter('sphinx')
140 >>> docwriter = ApiDocWriter('sphinx')
141 >>> import sphinx
141 >>> import sphinx
142 >>> docwriter.root_path == sphinx.__path__[0]
142 >>> docwriter.root_path == sphinx.__path__[0]
143 True
143 True
144 >>> docwriter.package_name = 'docutils'
144 >>> docwriter.package_name = 'docutils'
145 >>> import docutils
145 >>> import docutils
146 >>> docwriter.root_path == docutils.__path__[0]
146 >>> docwriter.root_path == docutils.__path__[0]
147 True
147 True
148 '''
148 '''
149 # It's also possible to imagine caching the module parsing here
149 # It's also possible to imagine caching the module parsing here
150 self._package_name = package_name
150 self._package_name = package_name
151 self.root_module = import_module(package_name)
151 self.root_module = import_module(package_name)
152 self.root_path = self.root_module.__path__[0]
152 self.root_path = self.root_module.__path__[0]
153 self.written_modules = None
153 self.written_modules = None
154
154
155 package_name = property(get_package_name, set_package_name, None,
155 package_name = property(get_package_name, set_package_name, None,
156 'get/set package_name')
156 'get/set package_name')
157
157
158 def _uri2path(self, uri):
158 def _uri2path(self, uri):
159 ''' Convert uri to absolute filepath
159 ''' Convert uri to absolute filepath
160
160
161 Parameters
161 Parameters
162 ----------
162 ----------
163 uri : string
163 uri : string
164 URI of python module to return path for
164 URI of python module to return path for
165
165
166 Returns
166 Returns
167 -------
167 -------
168 path : None or string
168 path : None or string
169 Returns None if there is no valid path for this URI
169 Returns None if there is no valid path for this URI
170 Otherwise returns absolute file system path for URI
170 Otherwise returns absolute file system path for URI
171
171
172 Examples
172 Examples
173 --------
173 --------
174 >>> docwriter = ApiDocWriter('sphinx')
174 >>> docwriter = ApiDocWriter('sphinx')
175 >>> import sphinx
175 >>> import sphinx
176 >>> modpath = sphinx.__path__[0]
176 >>> modpath = sphinx.__path__[0]
177 >>> res = docwriter._uri2path('sphinx.builder')
177 >>> res = docwriter._uri2path('sphinx.builder')
178 >>> res == os.path.join(modpath, 'builder.py')
178 >>> res == os.path.join(modpath, 'builder.py')
179 True
179 True
180 >>> res = docwriter._uri2path('sphinx')
180 >>> res = docwriter._uri2path('sphinx')
181 >>> res == os.path.join(modpath, '__init__.py')
181 >>> res == os.path.join(modpath, '__init__.py')
182 True
182 True
183 >>> docwriter._uri2path('sphinx.does_not_exist')
183 >>> docwriter._uri2path('sphinx.does_not_exist')
184
184
185 '''
185 '''
186 if uri == self.package_name:
186 if uri == self.package_name:
187 return os.path.join(self.root_path, '__init__.py')
187 return os.path.join(self.root_path, '__init__.py')
188 path = uri.replace('.', os.path.sep)
188 path = uri.replace('.', os.path.sep)
189 path = path.replace(self.package_name + os.path.sep, '')
189 path = path.replace(self.package_name + os.path.sep, '')
190 path = os.path.join(self.root_path, path)
190 path = os.path.join(self.root_path, path)
191 # XXX maybe check for extensions as well?
191 # XXX maybe check for extensions as well?
192 if os.path.exists(path + '.py'): # file
192 if os.path.exists(path + '.py'): # file
193 path += '.py'
193 path += '.py'
194 elif os.path.exists(os.path.join(path, '__init__.py')):
194 elif os.path.exists(os.path.join(path, '__init__.py')):
195 path = os.path.join(path, '__init__.py')
195 path = os.path.join(path, '__init__.py')
196 else:
196 else:
197 return None
197 return None
198 return path
198 return path
199
199
200 def _path2uri(self, dirpath):
200 def _path2uri(self, dirpath):
201 ''' Convert directory path to uri '''
201 ''' Convert directory path to uri '''
202 relpath = dirpath.replace(self.root_path, self.package_name)
202 relpath = dirpath.replace(self.root_path, self.package_name)
203 if relpath.startswith(os.path.sep):
203 if relpath.startswith(os.path.sep):
204 relpath = relpath[1:]
204 relpath = relpath[1:]
205 return relpath.replace(os.path.sep, '.')
205 return relpath.replace(os.path.sep, '.')
206
206
207 def _parse_module(self, uri):
207 def _parse_module(self, uri):
208 ''' Parse module defined in *uri* '''
208 ''' Parse module defined in *uri* '''
209 filename = self._uri2path(uri)
209 filename = self._uri2path(uri)
210 if filename is None:
210 if filename is None:
211 # nothing that we could handle here.
211 # nothing that we could handle here.
212 return ([],[])
212 return ([],[])
213 with open(filename, 'rb') as f:
213 with open(filename, 'rb') as f:
214 mod = ast.parse(f.read())
214 mod = ast.parse(f.read())
215 return FuncClsScanner().scan(mod)
215 return FuncClsScanner().scan(mod)
216
216
217 def _import_funcs_classes(self, uri):
217 def _import_funcs_classes(self, uri):
218 """Import * from uri, and separate out functions and classes."""
218 """Import * from uri, and separate out functions and classes."""
219 ns = {}
219 ns = {}
220 exec('from %s import *' % uri, ns)
220 exec('from %s import *' % uri, ns)
221 funcs, classes = [], []
221 funcs, classes = [], []
222 for name, obj in ns.items():
222 for name, obj in ns.items():
223 if inspect.isclass(obj):
223 if inspect.isclass(obj):
224 cls = Obj(name=name, has_init='__init__' in obj.__dict__)
224 cls = Obj(name=name, has_init='__init__' in obj.__dict__)
225 classes.append(cls)
225 classes.append(cls)
226 elif inspect.isfunction(obj):
226 elif inspect.isfunction(obj):
227 funcs.append(name)
227 funcs.append(name)
228
228
229 return sorted(funcs), sorted(classes, key=lambda x: x.name)
229 return sorted(funcs), sorted(classes, key=lambda x: x.name)
230
230
231 def find_funcs_classes(self, uri):
231 def find_funcs_classes(self, uri):
232 """Find the functions and classes defined in the module ``uri``"""
232 """Find the functions and classes defined in the module ``uri``"""
233 if uri in self.names_from__all__:
233 if uri in self.names_from__all__:
234 # For API modules which expose things defined elsewhere, import them
234 # For API modules which expose things defined elsewhere, import them
235 return self._import_funcs_classes(uri)
235 return self._import_funcs_classes(uri)
236 else:
236 else:
237 # For other modules, scan their AST to see what they define
237 # For other modules, scan their AST to see what they define
238 return self._parse_module(uri)
238 return self._parse_module(uri)
239
239
240 def generate_api_doc(self, uri):
240 def generate_api_doc(self, uri):
241 '''Make autodoc documentation template string for a module
241 '''Make autodoc documentation template string for a module
242
242
243 Parameters
243 Parameters
244 ----------
244 ----------
245 uri : string
245 uri : string
246 python location of module - e.g 'sphinx.builder'
246 python location of module - e.g 'sphinx.builder'
247
247
248 Returns
248 Returns
249 -------
249 -------
250 S : string
250 S : string
251 Contents of API doc
251 Contents of API doc
252 '''
252 '''
253 # get the names of all classes and functions
253 # get the names of all classes and functions
254 functions, classes = self.find_funcs_classes(uri)
254 functions, classes = self.find_funcs_classes(uri)
255 if not len(functions) and not len(classes):
255 if not len(functions) and not len(classes):
256 #print ('WARNING: Empty -', uri) # dbg
256 #print ('WARNING: Empty -', uri) # dbg
257 return ''
257 return ''
258
258
259 # Make a shorter version of the uri that omits the package name for
259 # Make a shorter version of the uri that omits the package name for
260 # titles
260 # titles
261 uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
261 uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
262
262
263 ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
263 ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
264
264
265 # Set the chapter title to read 'Module:' for all modules except for the
265 # Set the chapter title to read 'Module:' for all modules except for the
266 # main packages
266 # main packages
267 if '.' in uri:
267 if '.' in uri:
268 chap_title = 'Module: :mod:`' + uri_short + '`'
268 chap_title = 'Module: :mod:`' + uri_short + '`'
269 else:
269 else:
270 chap_title = ':mod:`' + uri_short + '`'
270 chap_title = ':mod:`' + uri_short + '`'
271 ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
271 ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)
272
272
273 ad += '\n.. automodule:: ' + uri + '\n'
273 ad += '\n.. automodule:: ' + uri + '\n'
274 ad += '\n.. currentmodule:: ' + uri + '\n'
274 ad += '\n.. currentmodule:: ' + uri + '\n'
275
275
276 if classes:
276 if classes:
277 subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
277 subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
278 ad += '\n'+ subhead + '\n' + \
278 ad += '\n'+ subhead + '\n' + \
279 self.rst_section_levels[2] * len(subhead) + '\n'
279 self.rst_section_levels[2] * len(subhead) + '\n'
280
280
281 for c in classes:
281 for c in classes:
282 ad += '\n.. autoclass:: ' + c.name + '\n'
282 ad += '\n.. autoclass:: ' + c.name + '\n'
283 # must NOT exclude from index to keep cross-refs working
283 # must NOT exclude from index to keep cross-refs working
284 ad += ' :members:\n' \
284 ad += ' :members:\n' \
285 ' :show-inheritance:\n'
285 ' :show-inheritance:\n'
286 if c.has_init:
286 if c.has_init:
287 ad += '\n .. automethod:: __init__\n'
287 ad += '\n .. automethod:: __init__\n'
288
288
289 if functions:
289 if functions:
290 subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
290 subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
291 ad += '\n'+ subhead + '\n' + \
291 ad += '\n'+ subhead + '\n' + \
292 self.rst_section_levels[2] * len(subhead) + '\n'
292 self.rst_section_levels[2] * len(subhead) + '\n'
293 for f in functions:
293 for f in functions:
294 # must NOT exclude from index to keep cross-refs working
294 # must NOT exclude from index to keep cross-refs working
295 ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
295 ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
296 return ad
296 return ad
297
297
298 def _survives_exclude(self, matchstr, match_type):
298 def _survives_exclude(self, matchstr, match_type):
299 ''' Returns True if *matchstr* does not match patterns
299 ''' Returns True if *matchstr* does not match patterns
300
300
301 ``self.package_name`` removed from front of string if present
301 ``self.package_name`` removed from front of string if present
302
302
303 Examples
303 Examples
304 --------
304 --------
305 >>> dw = ApiDocWriter('sphinx')
305 >>> dw = ApiDocWriter('sphinx')
306 >>> dw._survives_exclude('sphinx.okpkg', 'package')
306 >>> dw._survives_exclude('sphinx.okpkg', 'package')
307 True
307 True
308 >>> dw.package_skip_patterns.append('^\\.badpkg$')
308 >>> dw.package_skip_patterns.append('^\\.badpkg$')
309 >>> dw._survives_exclude('sphinx.badpkg', 'package')
309 >>> dw._survives_exclude('sphinx.badpkg', 'package')
310 False
310 False
311 >>> dw._survives_exclude('sphinx.badpkg', 'module')
311 >>> dw._survives_exclude('sphinx.badpkg', 'module')
312 True
312 True
313 >>> dw._survives_exclude('sphinx.badmod', 'module')
313 >>> dw._survives_exclude('sphinx.badmod', 'module')
314 True
314 True
315 >>> dw.module_skip_patterns.append('^\\.badmod$')
315 >>> dw.module_skip_patterns.append('^\\.badmod$')
316 >>> dw._survives_exclude('sphinx.badmod', 'module')
316 >>> dw._survives_exclude('sphinx.badmod', 'module')
317 False
317 False
318 '''
318 '''
319 if match_type == 'module':
319 if match_type == 'module':
320 patterns = self.module_skip_patterns
320 patterns = self.module_skip_patterns
321 elif match_type == 'package':
321 elif match_type == 'package':
322 patterns = self.package_skip_patterns
322 patterns = self.package_skip_patterns
323 else:
323 else:
324 raise ValueError('Cannot interpret match type "%s"'
324 raise ValueError('Cannot interpret match type "%s"'
325 % match_type)
325 % match_type)
326 # Match to URI without package name
326 # Match to URI without package name
327 L = len(self.package_name)
327 L = len(self.package_name)
328 if matchstr[:L] == self.package_name:
328 if matchstr[:L] == self.package_name:
329 matchstr = matchstr[L:]
329 matchstr = matchstr[L:]
330 for pat in patterns:
330 for pat in patterns:
331 try:
331 try:
332 pat.search
332 pat.search
333 except AttributeError:
333 except AttributeError:
334 pat = re.compile(pat)
334 pat = re.compile(pat)
335 if pat.search(matchstr):
335 if pat.search(matchstr):
336 return False
336 return False
337 return True
337 return True
338
338
339 def discover_modules(self):
339 def discover_modules(self):
340 ''' Return module sequence discovered from ``self.package_name``
340 ''' Return module sequence discovered from ``self.package_name``
341
341
342
342
343 Parameters
343 Parameters
344 ----------
344 ----------
345 None
345 None
346
346
347 Returns
347 Returns
348 -------
348 -------
349 mods : sequence
349 mods : sequence
350 Sequence of module names within ``self.package_name``
350 Sequence of module names within ``self.package_name``
351
351
352 Examples
352 Examples
353 --------
353 --------
354 >>> dw = ApiDocWriter('sphinx')
354 >>> dw = ApiDocWriter('sphinx')
355 >>> mods = dw.discover_modules()
355 >>> mods = dw.discover_modules()
356 >>> 'sphinx.util' in mods
356 >>> 'sphinx.util' in mods
357 True
357 True
358 >>> dw.package_skip_patterns.append('\.util$')
358 >>> dw.package_skip_patterns.append('\.util$')
359 >>> 'sphinx.util' in dw.discover_modules()
359 >>> 'sphinx.util' in dw.discover_modules()
360 False
360 False
361 >>>
361 >>>
362 '''
362 '''
363 modules = [self.package_name]
363 modules = [self.package_name]
364 # raw directory parsing
364 # raw directory parsing
365 for dirpath, dirnames, filenames in os.walk(self.root_path):
365 for dirpath, dirnames, filenames in os.walk(self.root_path):
366 # Check directory names for packages
366 # Check directory names for packages
367 root_uri = self._path2uri(os.path.join(self.root_path,
367 root_uri = self._path2uri(os.path.join(self.root_path,
368 dirpath))
368 dirpath))
369 for dirname in dirnames[:]: # copy list - we modify inplace
369 for dirname in dirnames[:]: # copy list - we modify inplace
370 package_uri = '.'.join((root_uri, dirname))
370 package_uri = '.'.join((root_uri, dirname))
371 if (self._uri2path(package_uri) and
371 if (self._uri2path(package_uri) and
372 self._survives_exclude(package_uri, 'package')):
372 self._survives_exclude(package_uri, 'package')):
373 modules.append(package_uri)
373 modules.append(package_uri)
374 else:
374 else:
375 dirnames.remove(dirname)
375 dirnames.remove(dirname)
376 # Check filenames for modules
376 # Check filenames for modules
377 for filename in filenames:
377 for filename in filenames:
378 module_name = filename[:-3]
378 module_name = filename[:-3]
379 module_uri = '.'.join((root_uri, module_name))
379 module_uri = '.'.join((root_uri, module_name))
380 if (self._uri2path(module_uri) and
380 if (self._uri2path(module_uri) and
381 self._survives_exclude(module_uri, 'module')):
381 self._survives_exclude(module_uri, 'module')):
382 modules.append(module_uri)
382 modules.append(module_uri)
383 return sorted(modules)
383 return sorted(modules)
384
384
385 def write_modules_api(self, modules,outdir):
385 def write_modules_api(self, modules,outdir):
386 # write the list
386 # write the list
387 written_modules = []
387 written_modules = []
388 for m in modules:
388 for m in modules:
389 api_str = self.generate_api_doc(m)
389 api_str = self.generate_api_doc(m)
390 if not api_str:
390 if not api_str:
391 continue
391 continue
392 # write out to file
392 # write out to file
393 outfile = os.path.join(outdir,
393 outfile = os.path.join(outdir,
394 m + self.rst_extension)
394 m + self.rst_extension)
395 fileobj = open(outfile, 'wt')
395 with open(outfile, 'wt') as fileobj:
396 fileobj.write(api_str)
396 fileobj.write(api_str)
397 fileobj.close()
398 written_modules.append(m)
397 written_modules.append(m)
399 self.written_modules = written_modules
398 self.written_modules = written_modules
400
399
401 def write_api_docs(self, outdir):
400 def write_api_docs(self, outdir):
402 """Generate API reST files.
401 """Generate API reST files.
403
402
404 Parameters
403 Parameters
405 ----------
404 ----------
406 outdir : string
405 outdir : string
407 Directory name in which to store files
406 Directory name in which to store files
408 We create automatic filenames for each module
407 We create automatic filenames for each module
409
408
410 Returns
409 Returns
411 -------
410 -------
412 None
411 None
413
412
414 Notes
413 Notes
415 -----
414 -----
416 Sets self.written_modules to list of written modules
415 Sets self.written_modules to list of written modules
417 """
416 """
418 if not os.path.exists(outdir):
417 if not os.path.exists(outdir):
419 os.mkdir(outdir)
418 os.mkdir(outdir)
420 # compose list of modules
419 # compose list of modules
421 modules = self.discover_modules()
420 modules = self.discover_modules()
422 self.write_modules_api(modules,outdir)
421 self.write_modules_api(modules,outdir)
423
422
424 def write_index(self, outdir, path='gen.rst', relative_to=None):
423 def write_index(self, outdir, path='gen.rst', relative_to=None):
425 """Make a reST API index file from written files
424 """Make a reST API index file from written files
426
425
427 Parameters
426 Parameters
428 ----------
427 ----------
429 outdir : string
428 outdir : string
430 Directory to which to write generated index file
429 Directory to which to write generated index file
431 path : string
430 path : string
432 Filename to write index to
431 Filename to write index to
433 relative_to : string
432 relative_to : string
434 path to which written filenames are relative. This
433 path to which written filenames are relative. This
435 component of the written file path will be removed from
434 component of the written file path will be removed from
436 outdir, in the generated index. Default is None, meaning,
435 outdir, in the generated index. Default is None, meaning,
437 leave path as it is.
436 leave path as it is.
438 """
437 """
439 if self.written_modules is None:
438 if self.written_modules is None:
440 raise ValueError('No modules written')
439 raise ValueError('No modules written')
441 # Get full filename path
440 # Get full filename path
442 path = os.path.join(outdir, path)
441 path = os.path.join(outdir, path)
443 # Path written into index is relative to rootpath
442 # Path written into index is relative to rootpath
444 if relative_to is not None:
443 if relative_to is not None:
445 relpath = outdir.replace(relative_to + os.path.sep, '')
444 relpath = outdir.replace(relative_to + os.path.sep, '')
446 else:
445 else:
447 relpath = outdir
446 relpath = outdir
448 idx = open(path,'wt')
447 with open(path,'wt') as idx:
449 w = idx.write
448 w = idx.write
450 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
449 w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
451 w('.. autosummary::\n'
450 w('.. autosummary::\n'
452 ' :toctree: %s\n\n' % relpath)
451 ' :toctree: %s\n\n' % relpath)
453 for mod in self.written_modules:
452 for mod in self.written_modules:
454 w(' %s\n' % mod)
453 w(' %s\n' % mod)
455 idx.close()
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now