##// END OF EJS Templates
Minor cleanups.
Fernando Perez -
Show More
@@ -1,601 +1,598 b''
1 """Nose Plugin that supports IPython doctests.
1 """Nose Plugin that supports IPython doctests.
2
2
3 Limitations:
3 Limitations:
4
4
5 - When generating examples for use as doctests, make sure that you have
5 - When generating examples for use as doctests, make sure that you have
6 pretty-printing OFF. This can be done either by starting ipython with the
6 pretty-printing OFF. This can be done either by starting ipython with the
7 flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
7 flag '--nopprint', by setting pprint to 0 in your ipythonrc file, or by
8 interactively disabling it with %Pprint. This is required so that IPython
8 interactively disabling it with %Pprint. This is required so that IPython
9 output matches that of normal Python, which is used by doctest for internal
9 output matches that of normal Python, which is used by doctest for internal
10 execution.
10 execution.
11
11
12 - Do not rely on specific prompt numbers for results (such as using
12 - Do not rely on specific prompt numbers for results (such as using
13 '_34==True', for example). For IPython tests run via an external process the
13 '_34==True', for example). For IPython tests run via an external process the
14 prompt numbers may be different, and IPython tests run as normal python code
14 prompt numbers may be different, and IPython tests run as normal python code
15 won't even have these special _NN variables set at all.
15 won't even have these special _NN variables set at all.
16
16
17 - IPython functions that produce output as a side-effect of calling a system
17 - IPython functions that produce output as a side-effect of calling a system
18 process (e.g. 'ls') can be doc-tested, but they must be handled in an
18 process (e.g. 'ls') can be doc-tested, but they must be handled in an
19 external IPython process. Such doctests must be tagged with:
19 external IPython process. Such doctests must be tagged with:
20
20
21 # ipdoctest: EXTERNAL
21 # ipdoctest: EXTERNAL
22
22
23 so that the testing machinery handles them differently. Since these are run
23 so that the testing machinery handles them differently. Since these are run
24 via pexpect in an external process, they can't deal with exceptions or other
24 via pexpect in an external process, they can't deal with exceptions or other
25 fancy featurs of regular doctests. You must limit such tests to simple
25 fancy featurs of regular doctests. You must limit such tests to simple
26 matching of the output. For this reason, I recommend you limit these kinds
26 matching of the output. For this reason, I recommend you limit these kinds
27 of doctests to features that truly require a separate process, and use the
27 of doctests to features that truly require a separate process, and use the
28 normal IPython ones (which have all the features of normal doctests) for
28 normal IPython ones (which have all the features of normal doctests) for
29 everything else. See the examples at the bottom of this file for a
29 everything else. See the examples at the bottom of this file for a
30 comparison of what can be done with both types.
30 comparison of what can be done with both types.
31 """
31 """
32
32
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Module imports
35 # Module imports
36
36
37 # From the standard library
37 # From the standard library
38 import __builtin__
38 import __builtin__
39 import commands
39 import commands
40 import doctest
40 import doctest
41 import inspect
41 import inspect
42 import logging
42 import logging
43 import os
43 import os
44 import re
44 import re
45 import sys
45 import sys
46 import unittest
46 import unittest
47
47
48 from inspect import getmodule
48 from inspect import getmodule
49
49
50 # Third-party modules
50 # Third-party modules
51 import nose.core
51 import nose.core
52
52
53 from nose.plugins import doctests, Plugin
53 from nose.plugins import doctests, Plugin
54 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
54 from nose.util import anyp, getpackage, test_address, resolve_name, tolist
55
55
56 # Our own imports
56 # Our own imports
57 #from extdoctest import ExtensionDoctest, DocTestFinder
57 #from extdoctest import ExtensionDoctest, DocTestFinder
58 #from dttools import DocTestFinder, DocTestCase
58 #from dttools import DocTestFinder, DocTestCase
59 #-----------------------------------------------------------------------------
59 #-----------------------------------------------------------------------------
60 # Module globals and other constants
60 # Module globals and other constants
61
61
62 log = logging.getLogger(__name__)
62 log = logging.getLogger(__name__)
63
63
64 ###########################################################################
64 ###########################################################################
65 # *** HACK ***
65 # *** HACK ***
66 # We must start our own ipython object and heavily muck with it so that all the
66 # We must start our own ipython object and heavily muck with it so that all the
67 # modifications IPython makes to system behavior don't send the doctest
67 # modifications IPython makes to system behavior don't send the doctest
68 # machinery into a fit. This code should be considered a gross hack, but it
68 # machinery into a fit. This code should be considered a gross hack, but it
69 # gets the job done.
69 # gets the job done.
70
70
71 def start_ipython():
71 def start_ipython():
72 """Start a global IPython shell, which we need for IPython-specific syntax.
72 """Start a global IPython shell, which we need for IPython-specific syntax.
73 """
73 """
74 import IPython
74 import IPython
75
75
76 def xsys(cmd):
76 def xsys(cmd):
77 """Execute a command and print its output.
77 """Execute a command and print its output.
78
78
79 This is just a convenience function to replace the IPython system call
79 This is just a convenience function to replace the IPython system call
80 with one that is more doctest-friendly.
80 with one that is more doctest-friendly.
81 """
81 """
82 cmd = _ip.IP.var_expand(cmd,depth=1)
82 cmd = _ip.IP.var_expand(cmd,depth=1)
83 sys.stdout.write(commands.getoutput(cmd))
83 sys.stdout.write(commands.getoutput(cmd))
84 sys.stdout.flush()
84 sys.stdout.flush()
85
85
86 # Store certain global objects that IPython modifies
86 # Store certain global objects that IPython modifies
87 _displayhook = sys.displayhook
87 _displayhook = sys.displayhook
88 _excepthook = sys.excepthook
88 _excepthook = sys.excepthook
89 _main = sys.modules.get('__main__')
89 _main = sys.modules.get('__main__')
90
90
91 # Start IPython instance
91 # Start IPython instance
92 IPython.Shell.IPShell(['--classic','--noterm_title'])
92 IPython.Shell.IPShell(['--classic','--noterm_title'])
93
93
94 # Deactivate the various python system hooks added by ipython for
94 # Deactivate the various python system hooks added by ipython for
95 # interactive convenience so we don't confuse the doctest system
95 # interactive convenience so we don't confuse the doctest system
96 sys.modules['__main__'] = _main
96 sys.modules['__main__'] = _main
97 sys.displayhook = _displayhook
97 sys.displayhook = _displayhook
98 sys.excepthook = _excepthook
98 sys.excepthook = _excepthook
99
99
100 # So that ipython magics and aliases can be doctested (they work by making
100 # So that ipython magics and aliases can be doctested (they work by making
101 # a call into a global _ip object)
101 # a call into a global _ip object)
102 _ip = IPython.ipapi.get()
102 _ip = IPython.ipapi.get()
103 __builtin__._ip = _ip
103 __builtin__._ip = _ip
104
104
105 # Modify the IPython system call with one that uses getoutput, so that we
105 # Modify the IPython system call with one that uses getoutput, so that we
106 # can capture subcommands and print them to Python's stdout, otherwise the
106 # can capture subcommands and print them to Python's stdout, otherwise the
107 # doctest machinery would miss them.
107 # doctest machinery would miss them.
108 _ip.system = xsys
108 _ip.system = xsys
109
109
110 # The start call MUST be made here. I'm not sure yet why it doesn't work if
110 # The start call MUST be made here. I'm not sure yet why it doesn't work if
111 # it is made later, at plugin initialization time, but in all my tests, that's
111 # it is made later, at plugin initialization time, but in all my tests, that's
112 # the case.
112 # the case.
113 start_ipython()
113 start_ipython()
114
114
115 # *** END HACK ***
115 # *** END HACK ***
116 ###########################################################################
116 ###########################################################################
117
117
118 #-----------------------------------------------------------------------------
118 # Classes and functions
119
120 def is_extension_module(filename):
121 """Return whether the given filename is an extension module.
122
123 This simply checks that the extension is either .so or .pyd.
124 """
125 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
126
127
119 # Modified version of the one in the stdlib, that fixes a python bug (doctests
128 # Modified version of the one in the stdlib, that fixes a python bug (doctests
120 # not found in extension modules, http://bugs.python.org/issue3158)
129 # not found in extension modules, http://bugs.python.org/issue3158)
121 class DocTestFinder(doctest.DocTestFinder):
130 class DocTestFinder(doctest.DocTestFinder):
122
131
123 def _from_module(self, module, object):
132 def _from_module(self, module, object):
124 """
133 """
125 Return true if the given object is defined in the given
134 Return true if the given object is defined in the given
126 module.
135 module.
127 """
136 """
128 if module is None:
137 if module is None:
129 #print '_fm C1' # dbg
138 #print '_fm C1' # dbg
130 return True
139 return True
131 elif inspect.isfunction(object):
140 elif inspect.isfunction(object):
132 #print '_fm C2' # dbg
141 #print '_fm C2' # dbg
133 return module.__dict__ is object.func_globals
142 return module.__dict__ is object.func_globals
134 elif inspect.isbuiltin(object):
143 elif inspect.isbuiltin(object):
135 #print '_fm C2-1' # dbg
144 #print '_fm C2-1' # dbg
136 return module.__name__ == object.__module__
145 return module.__name__ == object.__module__
137 elif inspect.isclass(object):
146 elif inspect.isclass(object):
138 #print '_fm C3' # dbg
147 #print '_fm C3' # dbg
139 return module.__name__ == object.__module__
148 return module.__name__ == object.__module__
140 elif inspect.ismethod(object):
149 elif inspect.ismethod(object):
141 # This one may be a bug in cython that fails to correctly set the
150 # This one may be a bug in cython that fails to correctly set the
142 # __module__ attribute of methods, but since the same error is easy
151 # __module__ attribute of methods, but since the same error is easy
143 # to make by extension code writers, having this safety in place
152 # to make by extension code writers, having this safety in place
144 # isn't such a bad idea
153 # isn't such a bad idea
145 #print '_fm C3-1' # dbg
154 #print '_fm C3-1' # dbg
146 return module.__name__ == object.im_class.__module__
155 return module.__name__ == object.im_class.__module__
147 elif inspect.getmodule(object) is not None:
156 elif inspect.getmodule(object) is not None:
148 #print '_fm C4' # dbg
157 #print '_fm C4' # dbg
149 #print 'C4 mod',module,'obj',object # dbg
158 #print 'C4 mod',module,'obj',object # dbg
150 return module is inspect.getmodule(object)
159 return module is inspect.getmodule(object)
151 elif hasattr(object, '__module__'):
160 elif hasattr(object, '__module__'):
152 #print '_fm C5' # dbg
161 #print '_fm C5' # dbg
153 return module.__name__ == object.__module__
162 return module.__name__ == object.__module__
154 elif isinstance(object, property):
163 elif isinstance(object, property):
155 #print '_fm C6' # dbg
164 #print '_fm C6' # dbg
156 return True # [XX] no way not be sure.
165 return True # [XX] no way not be sure.
157 else:
166 else:
158 raise ValueError("object must be a class or function")
167 raise ValueError("object must be a class or function")
159
168
160
161
162 def _find(self, tests, obj, name, module, source_lines, globs, seen):
169 def _find(self, tests, obj, name, module, source_lines, globs, seen):
163 """
170 """
164 Find tests for the given object and any contained objects, and
171 Find tests for the given object and any contained objects, and
165 add them to `tests`.
172 add them to `tests`.
166 """
173 """
167
174
168 doctest.DocTestFinder._find(self,tests, obj, name, module,
175 doctest.DocTestFinder._find(self,tests, obj, name, module,
169 source_lines, globs, seen)
176 source_lines, globs, seen)
170
177
171 # Below we re-run pieces of the above method with manual modifications,
178 # Below we re-run pieces of the above method with manual modifications,
172 # because the original code is buggy and fails to correctly identify
179 # because the original code is buggy and fails to correctly identify
173 # doctests in extension modules.
180 # doctests in extension modules.
174
181
175 # Local shorthands
182 # Local shorthands
176 from inspect import isroutine, isclass, ismodule
183 from inspect import isroutine, isclass, ismodule
177
184
178 # Look for tests in a module's contained objects.
185 # Look for tests in a module's contained objects.
179 if inspect.ismodule(obj) and self._recurse:
186 if inspect.ismodule(obj) and self._recurse:
180 for valname, val in obj.__dict__.items():
187 for valname, val in obj.__dict__.items():
181 valname1 = '%s.%s' % (name, valname)
188 valname1 = '%s.%s' % (name, valname)
182 if ( (isroutine(val) or isclass(val))
189 if ( (isroutine(val) or isclass(val))
183 and self._from_module(module, val) ):
190 and self._from_module(module, val) ):
184
191
185 self._find(tests, val, valname1, module, source_lines,
192 self._find(tests, val, valname1, module, source_lines,
186 globs, seen)
193 globs, seen)
187
194
188
189 # Look for tests in a class's contained objects.
195 # Look for tests in a class's contained objects.
190 if inspect.isclass(obj) and self._recurse:
196 if inspect.isclass(obj) and self._recurse:
191 #print 'RECURSE into class:',obj # dbg
197 #print 'RECURSE into class:',obj # dbg
192 for valname, val in obj.__dict__.items():
198 for valname, val in obj.__dict__.items():
193 #valname1 = '%s.%s' % (name, valname) # dbg
199 #valname1 = '%s.%s' % (name, valname) # dbg
194 #print 'N',name,'VN:',valname,'val:',str(val)[:77] # dbg
200 #print 'N',name,'VN:',valname,'val:',str(val)[:77] # dbg
195 # Special handling for staticmethod/classmethod.
201 # Special handling for staticmethod/classmethod.
196 if isinstance(val, staticmethod):
202 if isinstance(val, staticmethod):
197 val = getattr(obj, valname)
203 val = getattr(obj, valname)
198 if isinstance(val, classmethod):
204 if isinstance(val, classmethod):
199 val = getattr(obj, valname).im_func
205 val = getattr(obj, valname).im_func
200
206
201 # Recurse to methods, properties, and nested classes.
207 # Recurse to methods, properties, and nested classes.
202 if ((inspect.isfunction(val) or inspect.isclass(val) or
208 if ((inspect.isfunction(val) or inspect.isclass(val) or
203 inspect.ismethod(val) or
209 inspect.ismethod(val) or
204 isinstance(val, property)) and
210 isinstance(val, property)) and
205 self._from_module(module, val)):
211 self._from_module(module, val)):
206 valname = '%s.%s' % (name, valname)
212 valname = '%s.%s' % (name, valname)
207 self._find(tests, val, valname, module, source_lines,
213 self._find(tests, val, valname, module, source_lines,
208 globs, seen)
214 globs, seen)
209
215
210
216
211 class DocTestCase(doctests.DocTestCase):
217 class DocTestCase(doctests.DocTestCase):
212 """Proxy for DocTestCase: provides an address() method that
218 """Proxy for DocTestCase: provides an address() method that
213 returns the correct address for the doctest case. Otherwise
219 returns the correct address for the doctest case. Otherwise
214 acts as a proxy to the test case. To provide hints for address(),
220 acts as a proxy to the test case. To provide hints for address(),
215 an obj may also be passed -- this will be used as the test object
221 an obj may also be passed -- this will be used as the test object
216 for purposes of determining the test address, if it is provided.
222 for purposes of determining the test address, if it is provided.
217 """
223 """
218
224
219 # doctests loaded via find(obj) omit the module name
225 # doctests loaded via find(obj) omit the module name
220 # so we need to override id, __repr__ and shortDescription
226 # so we need to override id, __repr__ and shortDescription
221 # bonus: this will squash a 2.3 vs 2.4 incompatiblity
227 # bonus: this will squash a 2.3 vs 2.4 incompatiblity
222 def id(self):
228 def id(self):
223 name = self._dt_test.name
229 name = self._dt_test.name
224 filename = self._dt_test.filename
230 filename = self._dt_test.filename
225 if filename is not None:
231 if filename is not None:
226 pk = getpackage(filename)
232 pk = getpackage(filename)
227 if pk is not None and not name.startswith(pk):
233 if pk is not None and not name.startswith(pk):
228 name = "%s.%s" % (pk, name)
234 name = "%s.%s" % (pk, name)
229 return name
235 return name
230
236
231
237
232 # Classes and functions
233
234 def is_extension_module(filename):
235 """Return whether the given filename is an extension module.
236
237 This simply checks that the extension is either .so or .pyd.
238 """
239 return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
240
241
242 # A simple subclassing of the original with a different class name, so we can
238 # A simple subclassing of the original with a different class name, so we can
243 # distinguish and treat differently IPython examples from pure python ones.
239 # distinguish and treat differently IPython examples from pure python ones.
244 class IPExample(doctest.Example): pass
240 class IPExample(doctest.Example): pass
245
241
242
246 class IPExternalExample(doctest.Example):
243 class IPExternalExample(doctest.Example):
247 """Doctest examples to be run in an external process."""
244 """Doctest examples to be run in an external process."""
248
245
249 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
246 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
250 options=None):
247 options=None):
251 # Parent constructor
248 # Parent constructor
252 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
249 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
253
250
254 # An EXTRA newline is needed to prevent pexpect hangs
251 # An EXTRA newline is needed to prevent pexpect hangs
255 self.source += '\n'
252 self.source += '\n'
256
253
254
257 class IPDocTestParser(doctest.DocTestParser):
255 class IPDocTestParser(doctest.DocTestParser):
258 """
256 """
259 A class used to parse strings containing doctest examples.
257 A class used to parse strings containing doctest examples.
260
258
261 Note: This is a version modified to properly recognize IPython input and
259 Note: This is a version modified to properly recognize IPython input and
262 convert any IPython examples into valid Python ones.
260 convert any IPython examples into valid Python ones.
263 """
261 """
264 # This regular expression is used to find doctest examples in a
262 # This regular expression is used to find doctest examples in a
265 # string. It defines three groups: `source` is the source code
263 # string. It defines three groups: `source` is the source code
266 # (including leading indentation and prompts); `indent` is the
264 # (including leading indentation and prompts); `indent` is the
267 # indentation of the first (PS1) line of the source code; and
265 # indentation of the first (PS1) line of the source code; and
268 # `want` is the expected output (including leading indentation).
266 # `want` is the expected output (including leading indentation).
269
267
270 # Classic Python prompts or default IPython ones
268 # Classic Python prompts or default IPython ones
271 _PS1_PY = r'>>>'
269 _PS1_PY = r'>>>'
272 _PS2_PY = r'\.\.\.'
270 _PS2_PY = r'\.\.\.'
273
271
274 _PS1_IP = r'In\ \[\d+\]:'
272 _PS1_IP = r'In\ \[\d+\]:'
275 _PS2_IP = r'\ \ \ \.\.\.+:'
273 _PS2_IP = r'\ \ \ \.\.\.+:'
276
274
277 _RE_TPL = r'''
275 _RE_TPL = r'''
278 # Source consists of a PS1 line followed by zero or more PS2 lines.
276 # Source consists of a PS1 line followed by zero or more PS2 lines.
279 (?P<source>
277 (?P<source>
280 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
278 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
281 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
279 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
282 \n? # a newline
280 \n? # a newline
283 # Want consists of any non-blank lines that do not start with PS1.
281 # Want consists of any non-blank lines that do not start with PS1.
284 (?P<want> (?:(?![ ]*$) # Not a blank line
282 (?P<want> (?:(?![ ]*$) # Not a blank line
285 (?![ ]*%s) # Not a line starting with PS1
283 (?![ ]*%s) # Not a line starting with PS1
286 (?![ ]*%s) # Not a line starting with PS2
284 (?![ ]*%s) # Not a line starting with PS2
287 .*$\n? # But any other line
285 .*$\n? # But any other line
288 )*)
286 )*)
289 '''
287 '''
290
288
291 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
289 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
292 re.MULTILINE | re.VERBOSE)
290 re.MULTILINE | re.VERBOSE)
293
291
294 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
292 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
295 re.MULTILINE | re.VERBOSE)
293 re.MULTILINE | re.VERBOSE)
296
294
297 def ip2py(self,source):
295 def ip2py(self,source):
298 """Convert input IPython source into valid Python."""
296 """Convert input IPython source into valid Python."""
299 out = []
297 out = []
300 newline = out.append
298 newline = out.append
301 for lnum,line in enumerate(source.splitlines()):
299 for lnum,line in enumerate(source.splitlines()):
302 #newline(_ip.IPipython.prefilter(line,True))
300 #newline(_ip.IPipython.prefilter(line,True))
303 newline(_ip.IP.prefilter(line,lnum>0))
301 newline(_ip.IP.prefilter(line,lnum>0))
304 newline('') # ensure a closing newline, needed by doctest
302 newline('') # ensure a closing newline, needed by doctest
305 return '\n'.join(out)
303 return '\n'.join(out)
306
304
307 def parse(self, string, name='<string>'):
305 def parse(self, string, name='<string>'):
308 """
306 """
309 Divide the given string into examples and intervening text,
307 Divide the given string into examples and intervening text,
310 and return them as a list of alternating Examples and strings.
308 and return them as a list of alternating Examples and strings.
311 Line numbers for the Examples are 0-based. The optional
309 Line numbers for the Examples are 0-based. The optional
312 argument `name` is a name identifying this string, and is only
310 argument `name` is a name identifying this string, and is only
313 used for error messages.
311 used for error messages.
314 """
312 """
315
313
316 #print 'Parse string:\n',string # dbg
314 #print 'Parse string:\n',string # dbg
317
315
318 string = string.expandtabs()
316 string = string.expandtabs()
319 # If all lines begin with the same indentation, then strip it.
317 # If all lines begin with the same indentation, then strip it.
320 min_indent = self._min_indent(string)
318 min_indent = self._min_indent(string)
321 if min_indent > 0:
319 if min_indent > 0:
322 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
320 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
323
321
324 output = []
322 output = []
325 charno, lineno = 0, 0
323 charno, lineno = 0, 0
326
324
327 # Whether to convert the input from ipython to python syntax
325 # Whether to convert the input from ipython to python syntax
328 ip2py = False
326 ip2py = False
329 # Find all doctest examples in the string. First, try them as Python
327 # Find all doctest examples in the string. First, try them as Python
330 # examples, then as IPython ones
328 # examples, then as IPython ones
331 terms = list(self._EXAMPLE_RE_PY.finditer(string))
329 terms = list(self._EXAMPLE_RE_PY.finditer(string))
332 if terms:
330 if terms:
333 # Normal Python example
331 # Normal Python example
334 #print '-'*70 # dbg
332 #print '-'*70 # dbg
335 #print 'PyExample, Source:\n',string # dbg
333 #print 'PyExample, Source:\n',string # dbg
336 #print '-'*70 # dbg
334 #print '-'*70 # dbg
337 Example = doctest.Example
335 Example = doctest.Example
338 else:
336 else:
339 # It's an ipython example. Note that IPExamples are run
337 # It's an ipython example. Note that IPExamples are run
340 # in-process, so their syntax must be turned into valid python.
338 # in-process, so their syntax must be turned into valid python.
341 # IPExternalExamples are run out-of-process (via pexpect) so they
339 # IPExternalExamples are run out-of-process (via pexpect) so they
342 # don't need any filtering (a real ipython will be executing them).
340 # don't need any filtering (a real ipython will be executing them).
343 terms = list(self._EXAMPLE_RE_IP.finditer(string))
341 terms = list(self._EXAMPLE_RE_IP.finditer(string))
344 if re.search(r'#\s*ipdoctest:\s*EXTERNAL',string):
342 if re.search(r'#\s*ipdoctest:\s*EXTERNAL',string):
345 #print '-'*70 # dbg
343 #print '-'*70 # dbg
346 #print 'IPExternalExample, Source:\n',string # dbg
344 #print 'IPExternalExample, Source:\n',string # dbg
347 #print '-'*70 # dbg
345 #print '-'*70 # dbg
348 Example = IPExternalExample
346 Example = IPExternalExample
349 else:
347 else:
350 #print '-'*70 # dbg
348 #print '-'*70 # dbg
351 #print 'IPExample, Source:\n',string # dbg
349 #print 'IPExample, Source:\n',string # dbg
352 #print '-'*70 # dbg
350 #print '-'*70 # dbg
353 Example = IPExample
351 Example = IPExample
354 ip2py = True
352 ip2py = True
355
353
356 for m in terms:
354 for m in terms:
357 # Add the pre-example text to `output`.
355 # Add the pre-example text to `output`.
358 output.append(string[charno:m.start()])
356 output.append(string[charno:m.start()])
359 # Update lineno (lines before this example)
357 # Update lineno (lines before this example)
360 lineno += string.count('\n', charno, m.start())
358 lineno += string.count('\n', charno, m.start())
361 # Extract info from the regexp match.
359 # Extract info from the regexp match.
362 (source, options, want, exc_msg) = \
360 (source, options, want, exc_msg) = \
363 self._parse_example(m, name, lineno,ip2py)
361 self._parse_example(m, name, lineno,ip2py)
364 if Example is IPExternalExample:
362 if Example is IPExternalExample:
365 options[doctest.NORMALIZE_WHITESPACE] = True
363 options[doctest.NORMALIZE_WHITESPACE] = True
366 want += '\n'
364 want += '\n'
367 # Create an Example, and add it to the list.
365 # Create an Example, and add it to the list.
368 if not self._IS_BLANK_OR_COMMENT(source):
366 if not self._IS_BLANK_OR_COMMENT(source):
369 #print 'Example source:', source # dbg
367 #print 'Example source:', source # dbg
370 output.append(Example(source, want, exc_msg,
368 output.append(Example(source, want, exc_msg,
371 lineno=lineno,
369 lineno=lineno,
372 indent=min_indent+len(m.group('indent')),
370 indent=min_indent+len(m.group('indent')),
373 options=options))
371 options=options))
374 # Update lineno (lines inside this example)
372 # Update lineno (lines inside this example)
375 lineno += string.count('\n', m.start(), m.end())
373 lineno += string.count('\n', m.start(), m.end())
376 # Update charno.
374 # Update charno.
377 charno = m.end()
375 charno = m.end()
378 # Add any remaining post-example text to `output`.
376 # Add any remaining post-example text to `output`.
379 output.append(string[charno:])
377 output.append(string[charno:])
380
378
381 return output
379 return output
382
380
383 def _parse_example(self, m, name, lineno,ip2py=False):
381 def _parse_example(self, m, name, lineno,ip2py=False):
384 """
382 """
385 Given a regular expression match from `_EXAMPLE_RE` (`m`),
383 Given a regular expression match from `_EXAMPLE_RE` (`m`),
386 return a pair `(source, want)`, where `source` is the matched
384 return a pair `(source, want)`, where `source` is the matched
387 example's source code (with prompts and indentation stripped);
385 example's source code (with prompts and indentation stripped);
388 and `want` is the example's expected output (with indentation
386 and `want` is the example's expected output (with indentation
389 stripped).
387 stripped).
390
388
391 `name` is the string's name, and `lineno` is the line number
389 `name` is the string's name, and `lineno` is the line number
392 where the example starts; both are used for error messages.
390 where the example starts; both are used for error messages.
393
391
394 Optional:
392 Optional:
395 `ip2py`: if true, filter the input via IPython to convert the syntax
393 `ip2py`: if true, filter the input via IPython to convert the syntax
396 into valid python.
394 into valid python.
397 """
395 """
398
396
399 # Get the example's indentation level.
397 # Get the example's indentation level.
400 indent = len(m.group('indent'))
398 indent = len(m.group('indent'))
401
399
402 # Divide source into lines; check that they're properly
400 # Divide source into lines; check that they're properly
403 # indented; and then strip their indentation & prompts.
401 # indented; and then strip their indentation & prompts.
404 source_lines = m.group('source').split('\n')
402 source_lines = m.group('source').split('\n')
405
403
406 # We're using variable-length input prompts
404 # We're using variable-length input prompts
407 ps1 = m.group('ps1')
405 ps1 = m.group('ps1')
408 ps2 = m.group('ps2')
406 ps2 = m.group('ps2')
409 ps1_len = len(ps1)
407 ps1_len = len(ps1)
410
408
411 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
409 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
412 if ps2:
410 if ps2:
413 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
411 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
414
412
415 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
413 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
416
414
417 if ip2py:
415 if ip2py:
418 # Convert source input from IPython into valid Python syntax
416 # Convert source input from IPython into valid Python syntax
419 source = self.ip2py(source)
417 source = self.ip2py(source)
420
418
421 # Divide want into lines; check that it's properly indented; and
419 # Divide want into lines; check that it's properly indented; and
422 # then strip the indentation. Spaces before the last newline should
420 # then strip the indentation. Spaces before the last newline should
423 # be preserved, so plain rstrip() isn't good enough.
421 # be preserved, so plain rstrip() isn't good enough.
424 want = m.group('want')
422 want = m.group('want')
425 want_lines = want.split('\n')
423 want_lines = want.split('\n')
426 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
424 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
427 del want_lines[-1] # forget final newline & spaces after it
425 del want_lines[-1] # forget final newline & spaces after it
428 self._check_prefix(want_lines, ' '*indent, name,
426 self._check_prefix(want_lines, ' '*indent, name,
429 lineno + len(source_lines))
427 lineno + len(source_lines))
430
428
431 # Remove ipython output prompt that might be present in the first line
429 # Remove ipython output prompt that might be present in the first line
432 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
430 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
433
431
434 want = '\n'.join([wl[indent:] for wl in want_lines])
432 want = '\n'.join([wl[indent:] for wl in want_lines])
435
433
436 # If `want` contains a traceback message, then extract it.
434 # If `want` contains a traceback message, then extract it.
437 m = self._EXCEPTION_RE.match(want)
435 m = self._EXCEPTION_RE.match(want)
438 if m:
436 if m:
439 exc_msg = m.group('msg')
437 exc_msg = m.group('msg')
440 else:
438 else:
441 exc_msg = None
439 exc_msg = None
442
440
443 # Extract options from the source.
441 # Extract options from the source.
444 options = self._find_options(source, name, lineno)
442 options = self._find_options(source, name, lineno)
445
443
446 return source, options, want, exc_msg
444 return source, options, want, exc_msg
447
445
448 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
446 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
449 """
447 """
450 Given the lines of a source string (including prompts and
448 Given the lines of a source string (including prompts and
451 leading indentation), check to make sure that every prompt is
449 leading indentation), check to make sure that every prompt is
452 followed by a space character. If any line is not followed by
450 followed by a space character. If any line is not followed by
453 a space character, then raise ValueError.
451 a space character, then raise ValueError.
454
452
455 Note: IPython-modified version which takes the input prompt length as a
453 Note: IPython-modified version which takes the input prompt length as a
456 parameter, so that prompts of variable length can be dealt with.
454 parameter, so that prompts of variable length can be dealt with.
457 """
455 """
458 space_idx = indent+ps1_len
456 space_idx = indent+ps1_len
459 min_len = space_idx+1
457 min_len = space_idx+1
460 for i, line in enumerate(lines):
458 for i, line in enumerate(lines):
461 if len(line) >= min_len and line[space_idx] != ' ':
459 if len(line) >= min_len and line[space_idx] != ' ':
462 raise ValueError('line %r of the docstring for %s '
460 raise ValueError('line %r of the docstring for %s '
463 'lacks blank after %s: %r' %
461 'lacks blank after %s: %r' %
464 (lineno+i+1, name,
462 (lineno+i+1, name,
465 line[indent:space_idx], line))
463 line[indent:space_idx], line))
466
464
467 SKIP = doctest.register_optionflag('SKIP')
465 SKIP = doctest.register_optionflag('SKIP')
468
466
469 ###########################################################################
470
467
471 class DocFileCase(doctest.DocFileCase):
468 class DocFileCase(doctest.DocFileCase):
472 """Overrides to provide filename
469 """Overrides to provide filename
473 """
470 """
474 def address(self):
471 def address(self):
475 return (self._dt_test.filename, None, None)
472 return (self._dt_test.filename, None, None)
476
473
477
474
478 class ExtensionDoctest(doctests.Doctest):
475 class ExtensionDoctest(doctests.Doctest):
479 """Nose Plugin that supports doctests in extension modules.
476 """Nose Plugin that supports doctests in extension modules.
480 """
477 """
481 name = 'extdoctest' # call nosetests with --with-extdoctest
478 name = 'extdoctest' # call nosetests with --with-extdoctest
482 enabled = True
479 enabled = True
483
480
484 def options(self, parser, env=os.environ):
481 def options(self, parser, env=os.environ):
485 Plugin.options(self, parser, env)
482 Plugin.options(self, parser, env)
486
483
487 def configure(self, options, config):
484 def configure(self, options, config):
488 Plugin.configure(self, options, config)
485 Plugin.configure(self, options, config)
489 self.doctest_tests = options.doctest_tests
486 self.doctest_tests = options.doctest_tests
490 self.extension = tolist(options.doctestExtension)
487 self.extension = tolist(options.doctestExtension)
491 self.finder = DocTestFinder()
488 self.finder = DocTestFinder()
492 self.parser = doctest.DocTestParser()
489 self.parser = doctest.DocTestParser()
493
490
494
491
495 def loadTestsFromExtensionModule(self,filename):
492 def loadTestsFromExtensionModule(self,filename):
496 bpath,mod = os.path.split(filename)
493 bpath,mod = os.path.split(filename)
497 modname = os.path.splitext(mod)[0]
494 modname = os.path.splitext(mod)[0]
498 try:
495 try:
499 sys.path.append(bpath)
496 sys.path.append(bpath)
500 module = __import__(modname)
497 module = __import__(modname)
501 tests = list(self.loadTestsFromModule(module))
498 tests = list(self.loadTestsFromModule(module))
502 finally:
499 finally:
503 sys.path.pop()
500 sys.path.pop()
504 return tests
501 return tests
505
502
506 def loadTestsFromFile(self, filename):
503 def loadTestsFromFile(self, filename):
507 if is_extension_module(filename):
504 if is_extension_module(filename):
508 for t in self.loadTestsFromExtensionModule(filename):
505 for t in self.loadTestsFromExtensionModule(filename):
509 yield t
506 yield t
510 else:
507 else:
511 ## for t in list(doctests.Doctest.loadTestsFromFile(self,filename)):
508 ## for t in list(doctests.Doctest.loadTestsFromFile(self,filename)):
512 ## yield t
509 ## yield t
513 pass
510 pass
514
511
515 if self.extension and anyp(filename.endswith, self.extension):
512 if self.extension and anyp(filename.endswith, self.extension):
516 #print 'lTF',filename # dbg
513 #print 'lTF',filename # dbg
517 name = os.path.basename(filename)
514 name = os.path.basename(filename)
518 dh = open(filename)
515 dh = open(filename)
519 try:
516 try:
520 doc = dh.read()
517 doc = dh.read()
521 finally:
518 finally:
522 dh.close()
519 dh.close()
523 test = self.parser.get_doctest(
520 test = self.parser.get_doctest(
524 doc, globs={'__file__': filename}, name=name,
521 doc, globs={'__file__': filename}, name=name,
525 filename=filename, lineno=0)
522 filename=filename, lineno=0)
526 if test.examples:
523 if test.examples:
527 #print 'FileCase:',test.examples # dbg
524 #print 'FileCase:',test.examples # dbg
528 yield DocFileCase(test)
525 yield DocFileCase(test)
529 else:
526 else:
530 yield False # no tests to load
527 yield False # no tests to load
531
528
532
533 def wantFile(self,filename):
529 def wantFile(self,filename):
534 """Return whether the given filename should be scanned for tests.
530 """Return whether the given filename should be scanned for tests.
535
531
536 Modified version that accepts extension modules as valid containers for
532 Modified version that accepts extension modules as valid containers for
537 doctests.
533 doctests.
538 """
534 """
539 #print 'Filename:',filename # dbg
535 #print 'Filename:',filename # dbg
540
536
541 # temporarily hardcoded list, will move to driver later
537 # temporarily hardcoded list, will move to driver later
542 exclude = ['IPython/external/',
538 exclude = ['IPython/external/',
543 'IPython/Extensions/ipy_',
539 'IPython/Extensions/ipy_',
544 'IPython/platutils_win32',
540 'IPython/platutils_win32',
545 'IPython/frontend/cocoa',
541 'IPython/frontend/cocoa',
546 'IPython_doctest_plugin',
542 'IPython_doctest_plugin',
547 'IPython/Gnuplot',
543 'IPython/Gnuplot',
548 'IPython/Extensions/PhysicalQIn']
544 'IPython/Extensions/PhysicalQIn']
549
545
550 for fex in exclude:
546 for fex in exclude:
551 if fex in filename: # substring
547 if fex in filename: # substring
552 #print '###>>> SKIP:',filename # dbg
548 #print '###>>> SKIP:',filename # dbg
553 return False
549 return False
554
550
555 if is_extension_module(filename):
551 if is_extension_module(filename):
556 return True
552 return True
557 else:
553 else:
558 return doctests.Doctest.wantFile(self,filename)
554 return doctests.Doctest.wantFile(self,filename)
559
555
560 # NOTE: the method below is a *copy* of the one in the nose doctests
556 # NOTE: the method below is a *copy* of the one in the nose doctests
561 # plugin, but we have to replicate it here in order to have it resolve the
557 # plugin, but we have to replicate it here in order to have it resolve the
562 # DocTestCase (last line) to our local copy, since the nose plugin doesn't
558 # DocTestCase (last line) to our local copy, since the nose plugin doesn't
563 # provide a public hook for what TestCase class to use. The alternative
559 # provide a public hook for what TestCase class to use. The alternative
564 # would be to monkeypatch doctest in the stdlib, but that's ugly and
560 # would be to monkeypatch doctest in the stdlib, but that's ugly and
565 # brittle, since a change in plugin load order can break it. So for now,
561 # brittle, since a change in plugin load order can break it. So for now,
566 # we just paste this in here, inelegant as this may be.
562 # we just paste this in here, inelegant as this may be.
567
563
568 def loadTestsFromModule(self, module):
564 def loadTestsFromModule(self, module):
569 #print 'lTM',module # dbg
565 #print 'lTM',module # dbg
570
566
571 if not self.matches(module.__name__):
567 if not self.matches(module.__name__):
572 log.debug("Doctest doesn't want module %s", module)
568 log.debug("Doctest doesn't want module %s", module)
573 return
569 return
574 tests = self.finder.find(module)
570 tests = self.finder.find(module)
575 if not tests:
571 if not tests:
576 return
572 return
577 tests.sort()
573 tests.sort()
578 module_file = module.__file__
574 module_file = module.__file__
579 if module_file[-4:] in ('.pyc', '.pyo'):
575 if module_file[-4:] in ('.pyc', '.pyo'):
580 module_file = module_file[:-1]
576 module_file = module_file[:-1]
581 for test in tests:
577 for test in tests:
582 if not test.examples:
578 if not test.examples:
583 continue
579 continue
584 if not test.filename:
580 if not test.filename:
585 test.filename = module_file
581 test.filename = module_file
586 yield DocTestCase(test)
582 yield DocTestCase(test)
587
583
584
588 class IPythonDoctest(ExtensionDoctest):
585 class IPythonDoctest(ExtensionDoctest):
589 """Nose Plugin that supports doctests in extension modules.
586 """Nose Plugin that supports doctests in extension modules.
590 """
587 """
591 name = 'ipdoctest' # call nosetests with --with-ipdoctest
588 name = 'ipdoctest' # call nosetests with --with-ipdoctest
592 enabled = True
589 enabled = True
593
590
594 def configure(self, options, config):
591 def configure(self, options, config):
595
592
596 Plugin.configure(self, options, config)
593 Plugin.configure(self, options, config)
597 self.doctest_tests = options.doctest_tests
594 self.doctest_tests = options.doctest_tests
598 self.extension = tolist(options.doctestExtension)
595 self.extension = tolist(options.doctestExtension)
599 self.parser = IPDocTestParser()
596 self.parser = IPDocTestParser()
600 #self.finder = DocTestFinder(parser=IPDocTestParser())
597 #self.finder = DocTestFinder(parser=IPDocTestParser())
601 self.finder = DocTestFinder(parser=self.parser)
598 self.finder = DocTestFinder(parser=self.parser)
General Comments 0
You need to be logged in to leave comments. Login now