##// END OF EJS Templates
Remove unneeded workaround...
Nikita Kniazev -
Show More
@@ -1,346 +1,334
1 1 """Nose Plugin that supports IPython doctests.
2 2
3 3 Limitations:
4 4
5 5 - When generating examples for use as doctests, make sure that you have
6 6 pretty-printing OFF. This can be done either by setting the
7 7 ``PlainTextFormatter.pprint`` option in your configuration file to False, or
8 8 by interactively disabling it with %Pprint. This is required so that IPython
9 9 output matches that of normal Python, which is used by doctest for internal
10 10 execution.
11 11
12 12 - Do not rely on specific prompt numbers for results (such as using
13 13 '_34==True', for example). For IPython tests run via an external process the
14 14 prompt numbers may be different, and IPython tests run as normal python code
15 15 won't even have these special _NN variables set at all.
16 16 """
17 17
18 18 #-----------------------------------------------------------------------------
19 19 # Module imports
20 20
21 21 # From the standard library
22 22 import doctest
23 23 import logging
24 24 import os
25 25 import re
26 26
27 27 from testpath import modified_env
28 28
29 29 #-----------------------------------------------------------------------------
30 30 # Module globals and other constants
31 31 #-----------------------------------------------------------------------------
32 32
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 36 #-----------------------------------------------------------------------------
37 37 # Classes and functions
38 38 #-----------------------------------------------------------------------------
39 39
40 40
41 41 class DocTestFinder(doctest.DocTestFinder):
42 42 def _get_test(self, obj, name, module, globs, source_lines):
43 43 test = super()._get_test(obj, name, module, globs, source_lines)
44 44
45 45 if bool(getattr(obj, "__skip_doctest__", False)) and test is not None:
46 46 for example in test.examples:
47 47 example.options[doctest.SKIP] = True
48 48
49 49 return test
50 50
51 51
52 52 class IPDoctestOutputChecker(doctest.OutputChecker):
53 53 """Second-chance checker with support for random tests.
54 54
55 55 If the default comparison doesn't pass, this checker looks in the expected
56 56 output string for flags that tell us to ignore the output.
57 57 """
58 58
59 59 random_re = re.compile(r'#\s*random\s+')
60 60
61 61 def check_output(self, want, got, optionflags):
62 62 """Check output, accepting special markers embedded in the output.
63 63
64 64 If the output didn't pass the default validation but the special string
65 65 '#random' is included, we accept it."""
66 66
67 67 # Let the original tester verify first, in case people have valid tests
68 68 # that happen to have a comment saying '#random' embedded in.
69 69 ret = doctest.OutputChecker.check_output(self, want, got,
70 70 optionflags)
71 71 if not ret and self.random_re.search(want):
72 72 #print >> sys.stderr, 'RANDOM OK:',want # dbg
73 73 return True
74 74
75 75 return ret
76 76
77 77
78 78 # A simple subclassing of the original with a different class name, so we can
79 79 # distinguish and treat differently IPython examples from pure python ones.
80 80 class IPExample(doctest.Example): pass
81 81
82 82
83 83 class IPExternalExample(doctest.Example):
84 84 """Doctest examples to be run in an external process."""
85 85
86 86 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
87 87 options=None):
88 88 # Parent constructor
89 89 doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
90 90
91 91 # An EXTRA newline is needed to prevent pexpect hangs
92 92 self.source += '\n'
93 93
94 94
95 95 class IPDocTestParser(doctest.DocTestParser):
96 96 """
97 97 A class used to parse strings containing doctest examples.
98 98
99 99 Note: This is a version modified to properly recognize IPython input and
100 100 convert any IPython examples into valid Python ones.
101 101 """
102 102 # This regular expression is used to find doctest examples in a
103 103 # string. It defines three groups: `source` is the source code
104 104 # (including leading indentation and prompts); `indent` is the
105 105 # indentation of the first (PS1) line of the source code; and
106 106 # `want` is the expected output (including leading indentation).
107 107
108 108 # Classic Python prompts or default IPython ones
109 109 _PS1_PY = r'>>>'
110 110 _PS2_PY = r'\.\.\.'
111 111
112 112 _PS1_IP = r'In\ \[\d+\]:'
113 113 _PS2_IP = r'\ \ \ \.\.\.+:'
114 114
115 115 _RE_TPL = r'''
116 116 # Source consists of a PS1 line followed by zero or more PS2 lines.
117 117 (?P<source>
118 118 (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*) # PS1 line
119 119 (?:\n [ ]* (?P<ps2> %s) .*)*) # PS2 lines
120 120 \n? # a newline
121 121 # Want consists of any non-blank lines that do not start with PS1.
122 122 (?P<want> (?:(?![ ]*$) # Not a blank line
123 123 (?![ ]*%s) # Not a line starting with PS1
124 124 (?![ ]*%s) # Not a line starting with PS2
125 125 .*$\n? # But any other line
126 126 )*)
127 127 '''
128 128
129 129 _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
130 130 re.MULTILINE | re.VERBOSE)
131 131
132 132 _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
133 133 re.MULTILINE | re.VERBOSE)
134 134
135 135 # Mark a test as being fully random. In this case, we simply append the
136 136 # random marker ('#random') to each individual example's output. This way
137 137 # we don't need to modify any other code.
138 138 _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
139 139
140 140 # Mark tests to be executed in an external process - currently unsupported.
141 141 _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
142 142
143 143 def ip2py(self,source):
144 144 """Convert input IPython source into valid Python."""
145 145 block = _ip.input_transformer_manager.transform_cell(source)
146 146 if len(block.splitlines()) == 1:
147 147 return _ip.prefilter(block)
148 148 else:
149 149 return block
150 150
151 151 def parse(self, string, name='<string>'):
152 152 """
153 153 Divide the given string into examples and intervening text,
154 154 and return them as a list of alternating Examples and strings.
155 155 Line numbers for the Examples are 0-based. The optional
156 156 argument `name` is a name identifying this string, and is only
157 157 used for error messages.
158 158 """
159 159
160 160 #print 'Parse string:\n',string # dbg
161 161
162 162 string = string.expandtabs()
163 163 # If all lines begin with the same indentation, then strip it.
164 164 min_indent = self._min_indent(string)
165 165 if min_indent > 0:
166 166 string = '\n'.join([l[min_indent:] for l in string.split('\n')])
167 167
168 168 output = []
169 169 charno, lineno = 0, 0
170 170
171 171 # We make 'all random' tests by adding the '# random' mark to every
172 172 # block of output in the test.
173 173 if self._RANDOM_TEST.search(string):
174 174 random_marker = '\n# random'
175 175 else:
176 176 random_marker = ''
177 177
178 178 # Whether to convert the input from ipython to python syntax
179 179 ip2py = False
180 180 # Find all doctest examples in the string. First, try them as Python
181 181 # examples, then as IPython ones
182 182 terms = list(self._EXAMPLE_RE_PY.finditer(string))
183 183 if terms:
184 184 # Normal Python example
185 185 #print '-'*70 # dbg
186 186 #print 'PyExample, Source:\n',string # dbg
187 187 #print '-'*70 # dbg
188 188 Example = doctest.Example
189 189 else:
190 190 # It's an ipython example. Note that IPExamples are run
191 191 # in-process, so their syntax must be turned into valid python.
192 192 # IPExternalExamples are run out-of-process (via pexpect) so they
193 193 # don't need any filtering (a real ipython will be executing them).
194 194 terms = list(self._EXAMPLE_RE_IP.finditer(string))
195 195 if self._EXTERNAL_IP.search(string):
196 196 #print '-'*70 # dbg
197 197 #print 'IPExternalExample, Source:\n',string # dbg
198 198 #print '-'*70 # dbg
199 199 Example = IPExternalExample
200 200 else:
201 201 #print '-'*70 # dbg
202 202 #print 'IPExample, Source:\n',string # dbg
203 203 #print '-'*70 # dbg
204 204 Example = IPExample
205 205 ip2py = True
206 206
207 207 for m in terms:
208 208 # Add the pre-example text to `output`.
209 209 output.append(string[charno:m.start()])
210 210 # Update lineno (lines before this example)
211 211 lineno += string.count('\n', charno, m.start())
212 212 # Extract info from the regexp match.
213 213 (source, options, want, exc_msg) = \
214 214 self._parse_example(m, name, lineno,ip2py)
215 215
216 216 # Append the random-output marker (it defaults to empty in most
217 217 # cases, it's only non-empty for 'all-random' tests):
218 218 want += random_marker
219 219
220 220 if Example is IPExternalExample:
221 221 options[doctest.NORMALIZE_WHITESPACE] = True
222 222 want += '\n'
223 223
224 224 # Create an Example, and add it to the list.
225 225 if not self._IS_BLANK_OR_COMMENT(source):
226 226 output.append(Example(source, want, exc_msg,
227 227 lineno=lineno,
228 228 indent=min_indent+len(m.group('indent')),
229 229 options=options))
230 230 # Update lineno (lines inside this example)
231 231 lineno += string.count('\n', m.start(), m.end())
232 232 # Update charno.
233 233 charno = m.end()
234 234 # Add any remaining post-example text to `output`.
235 235 output.append(string[charno:])
236 236 return output
237 237
238 238 def _parse_example(self, m, name, lineno,ip2py=False):
239 239 """
240 240 Given a regular expression match from `_EXAMPLE_RE` (`m`),
241 241 return a pair `(source, want)`, where `source` is the matched
242 242 example's source code (with prompts and indentation stripped);
243 243 and `want` is the example's expected output (with indentation
244 244 stripped).
245 245
246 246 `name` is the string's name, and `lineno` is the line number
247 247 where the example starts; both are used for error messages.
248 248
249 249 Optional:
250 250 `ip2py`: if true, filter the input via IPython to convert the syntax
251 251 into valid python.
252 252 """
253 253
254 254 # Get the example's indentation level.
255 255 indent = len(m.group('indent'))
256 256
257 257 # Divide source into lines; check that they're properly
258 258 # indented; and then strip their indentation & prompts.
259 259 source_lines = m.group('source').split('\n')
260 260
261 261 # We're using variable-length input prompts
262 262 ps1 = m.group('ps1')
263 263 ps2 = m.group('ps2')
264 264 ps1_len = len(ps1)
265 265
266 266 self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
267 267 if ps2:
268 268 self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
269 269
270 270 source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
271 271
272 272 if ip2py:
273 273 # Convert source input from IPython into valid Python syntax
274 274 source = self.ip2py(source)
275 275
276 276 # Divide want into lines; check that it's properly indented; and
277 277 # then strip the indentation. Spaces before the last newline should
278 278 # be preserved, so plain rstrip() isn't good enough.
279 279 want = m.group('want')
280 280 want_lines = want.split('\n')
281 281 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
282 282 del want_lines[-1] # forget final newline & spaces after it
283 283 self._check_prefix(want_lines, ' '*indent, name,
284 284 lineno + len(source_lines))
285 285
286 286 # Remove ipython output prompt that might be present in the first line
287 287 want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
288 288
289 289 want = '\n'.join([wl[indent:] for wl in want_lines])
290 290
291 291 # If `want` contains a traceback message, then extract it.
292 292 m = self._EXCEPTION_RE.match(want)
293 293 if m:
294 294 exc_msg = m.group('msg')
295 295 else:
296 296 exc_msg = None
297 297
298 298 # Extract options from the source.
299 299 options = self._find_options(source, name, lineno)
300 300
301 301 return source, options, want, exc_msg
302 302
303 303 def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
304 304 """
305 305 Given the lines of a source string (including prompts and
306 306 leading indentation), check to make sure that every prompt is
307 307 followed by a space character. If any line is not followed by
308 308 a space character, then raise ValueError.
309 309
310 310 Note: IPython-modified version which takes the input prompt length as a
311 311 parameter, so that prompts of variable length can be dealt with.
312 312 """
313 313 space_idx = indent+ps1_len
314 314 min_len = space_idx+1
315 315 for i, line in enumerate(lines):
316 316 if len(line) >= min_len and line[space_idx] != ' ':
317 317 raise ValueError('line %r of the docstring for %s '
318 318 'lacks blank after %s: %r' %
319 319 (lineno+i+1, name,
320 320 line[indent:space_idx], line))
321 321
322 322
323 323 SKIP = doctest.register_optionflag('SKIP')
324 324
325 325
326 326 class IPDocTestRunner(doctest.DocTestRunner,object):
327 327 """Test runner that synchronizes the IPython namespace with test globals.
328 328 """
329 329
330 330 def run(self, test, compileflags=None, out=None, clear_globs=True):
331
332 # Hack: ipython needs access to the execution context of the example,
333 # so that it can propagate user variables loaded by %run into
334 # test.globs. We put them here into our modified %run as a function
335 # attribute. Our new %run will then only make the namespace update
336 # when called (rather than unconditionally updating test.globs here
337 # for all examples, most of which won't be calling %run anyway).
338 #_ip._ipdoctest_test_globs = test.globs
339 #_ip._ipdoctest_test_filename = test.filename
340
341 test.globs.update(_ip.user_ns)
342
343 331 # Override terminal size to standardise traceback format
344 332 with modified_env({'COLUMNS': '80', 'LINES': '24'}):
345 333 return super(IPDocTestRunner,self).run(test,
346 334 compileflags,out,clear_globs)
General Comments 0
You need to be logged in to leave comments. Login now