Show More
@@ -1,638 +1,646 b'' | |||
|
1 | 1 | """Input handling and transformation machinery. |
|
2 | 2 | |
|
3 | 3 | The first class in this module, :class:`InputSplitter`, is designed to tell when |
|
4 | 4 | input from a line-oriented frontend is complete and should be executed, and when |
|
5 | 5 | the user should be prompted for another line of code instead. The name 'input |
|
6 | 6 | splitter' is largely for historical reasons. |
|
7 | 7 | |
|
8 | 8 | A companion, :class:`IPythonInputSplitter`, provides the same functionality but |
|
9 | 9 | with full support for the extended IPython syntax (magics, system calls, etc). |
|
10 | 10 | The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`. |
|
11 | 11 | :class:`IPythonInputSplitter` feeds the raw code to the transformers in order |
|
12 | 12 | and stores the results. |
|
13 | 13 | |
|
14 | 14 | For more details, see the class docstrings below. |
|
15 | ||
|
16 | Authors | |
|
17 | ------- | |
|
18 | ||
|
19 | * Fernando Perez | |
|
20 | * Brian Granger | |
|
21 | * Thomas Kluyver | |
|
22 | 15 | """ |
|
23 | #----------------------------------------------------------------------------- | |
|
24 | # Copyright (C) 2010 The IPython Development Team | |
|
25 | # | |
|
26 | # Distributed under the terms of the BSD License. The full license is in | |
|
27 | # the file COPYING, distributed as part of this software. | |
|
28 | #----------------------------------------------------------------------------- | |
|
29 | 16 | |
|
30 | #----------------------------------------------------------------------------- | |
|
31 | # Imports | |
|
32 | #----------------------------------------------------------------------------- | |
|
33 | # stdlib | |
|
17 | # Copyright (c) IPython Development Team. | |
|
18 | # Distributed under the terms of the Modified BSD License. | |
|
34 | 19 | import ast |
|
35 | 20 | import codeop |
|
36 | 21 | import re |
|
37 | 22 | import sys |
|
38 | 23 | |
|
39 | # IPython modules | |
|
40 | 24 | from IPython.utils.py3compat import cast_unicode |
|
41 | 25 | from IPython.core.inputtransformer import (leading_indent, |
|
42 | 26 | classic_prompt, |
|
43 | 27 | ipy_prompt, |
|
44 | 28 | strip_encoding_cookie, |
|
45 | 29 | cellmagic, |
|
46 | 30 | assemble_logical_lines, |
|
47 | 31 | help_end, |
|
48 | 32 | escaped_commands, |
|
49 | 33 | assign_from_magic, |
|
50 | 34 | assign_from_system, |
|
51 | 35 | assemble_python_lines, |
|
52 | 36 | ) |
|
53 | 37 | |
|
54 | 38 | # These are available in this module for backwards compatibility. |
|
55 | 39 | from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP, |
|
56 | 40 | ESC_HELP2, ESC_MAGIC, ESC_MAGIC2, |
|
57 | 41 | ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES) |
|
58 | 42 | |
|
59 | 43 | #----------------------------------------------------------------------------- |
|
60 | 44 | # Utilities |
|
61 | 45 | #----------------------------------------------------------------------------- |
|
62 | 46 | |
|
63 | 47 | # FIXME: These are general-purpose utilities that later can be moved to the |
|
64 | 48 | # general ward. Kept here for now because we're being very strict about test |
|
65 | 49 | # coverage with this code, and this lets us ensure that we keep 100% coverage |
|
66 | 50 | # while developing. |
|
67 | 51 | |
|
68 | 52 | # compiled regexps for autoindent management |
|
69 | 53 | dedent_re = re.compile('|'.join([ |
|
70 | 54 | r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe) |
|
71 | 55 | r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren |
|
72 | 56 | r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe) |
|
73 | 57 | r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren |
|
74 | 58 | r'^\s+pass\s*$', # pass (optionally followed by trailing spaces) |
|
75 | 59 | r'^\s+break\s*$', # break (optionally followed by trailing spaces) |
|
76 | 60 | r'^\s+continue\s*$', # continue (optionally followed by trailing spaces) |
|
77 | 61 | ])) |
|
78 | 62 | ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)') |
|
79 | 63 | |
|
80 | 64 | # regexp to match pure comment lines so we don't accidentally insert 'if 1:' |
|
81 | 65 | # before pure comments |
|
82 | 66 | comment_line_re = re.compile('^\s*\#') |
|
83 | 67 | |
|
84 | 68 | |
|
85 | 69 | def num_ini_spaces(s): |
|
86 | 70 | """Return the number of initial spaces in a string. |
|
87 | 71 | |
|
88 | 72 | Note that tabs are counted as a single space. For now, we do *not* support |
|
89 | 73 | mixing of tabs and spaces in the user's input. |
|
90 | 74 | |
|
91 | 75 | Parameters |
|
92 | 76 | ---------- |
|
93 | 77 | s : string |
|
94 | 78 | |
|
95 | 79 | Returns |
|
96 | 80 | ------- |
|
97 | 81 | n : int |
|
98 | 82 | """ |
|
99 | 83 | |
|
100 | 84 | ini_spaces = ini_spaces_re.match(s) |
|
101 | 85 | if ini_spaces: |
|
102 | 86 | return ini_spaces.end() |
|
103 | 87 | else: |
|
104 | 88 | return 0 |
|
105 | 89 | |
|
106 | 90 | def last_blank(src): |
|
107 | 91 | """Determine if the input source ends in a blank. |
|
108 | 92 | |
|
109 | 93 | A blank is either a newline or a line consisting of whitespace. |
|
110 | 94 | |
|
111 | 95 | Parameters |
|
112 | 96 | ---------- |
|
113 | 97 | src : string |
|
114 | 98 | A single or multiline string. |
|
115 | 99 | """ |
|
116 | 100 | if not src: return False |
|
117 | 101 | ll = src.splitlines()[-1] |
|
118 | 102 | return (ll == '') or ll.isspace() |
|
119 | 103 | |
|
120 | 104 | |
|
121 | 105 | last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE) |
|
122 | 106 | last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE) |
|
123 | 107 | |
|
124 | 108 | def last_two_blanks(src): |
|
125 | 109 | """Determine if the input source ends in two blanks. |
|
126 | 110 | |
|
127 | 111 | A blank is either a newline or a line consisting of whitespace. |
|
128 | 112 | |
|
129 | 113 | Parameters |
|
130 | 114 | ---------- |
|
131 | 115 | src : string |
|
132 | 116 | A single or multiline string. |
|
133 | 117 | """ |
|
134 | 118 | if not src: return False |
|
135 | 119 | # The logic here is tricky: I couldn't get a regexp to work and pass all |
|
136 | 120 | # the tests, so I took a different approach: split the source by lines, |
|
137 | 121 | # grab the last two and prepend '###\n' as a stand-in for whatever was in |
|
138 | 122 | # the body before the last two lines. Then, with that structure, it's |
|
139 | 123 | # possible to analyze with two regexps. Not the most elegant solution, but |
|
140 | 124 | # it works. If anyone tries to change this logic, make sure to validate |
|
141 | 125 | # the whole test suite first! |
|
142 | 126 | new_src = '\n'.join(['###\n'] + src.splitlines()[-2:]) |
|
143 | 127 | return (bool(last_two_blanks_re.match(new_src)) or |
|
144 | 128 | bool(last_two_blanks_re2.match(new_src)) ) |
|
145 | 129 | |
|
146 | 130 | |
|
147 | 131 | def remove_comments(src): |
|
148 | 132 | """Remove all comments from input source. |
|
149 | 133 | |
|
150 | 134 | Note: comments are NOT recognized inside of strings! |
|
151 | 135 | |
|
152 | 136 | Parameters |
|
153 | 137 | ---------- |
|
154 | 138 | src : string |
|
155 | 139 | A single or multiline input string. |
|
156 | 140 | |
|
157 | 141 | Returns |
|
158 | 142 | ------- |
|
159 | 143 | String with all Python comments removed. |
|
160 | 144 | """ |
|
161 | 145 | |
|
162 | 146 | return re.sub('#.*', '', src) |
|
163 | 147 | |
|
164 | 148 | |
|
165 | 149 | def get_input_encoding(): |
|
166 | 150 | """Return the default standard input encoding. |
|
167 | 151 | |
|
168 | 152 | If sys.stdin has no encoding, 'ascii' is returned.""" |
|
169 | 153 | # There are strange environments for which sys.stdin.encoding is None. We |
|
170 | 154 | # ensure that a valid encoding is returned. |
|
171 | 155 | encoding = getattr(sys.stdin, 'encoding', None) |
|
172 | 156 | if encoding is None: |
|
173 | 157 | encoding = 'ascii' |
|
174 | 158 | return encoding |
|
175 | 159 | |
|
176 | 160 | #----------------------------------------------------------------------------- |
|
177 | 161 | # Classes and functions for normal Python syntax handling |
|
178 | 162 | #----------------------------------------------------------------------------- |
|
179 | 163 | |
|
180 | 164 | class InputSplitter(object): |
|
181 | 165 | r"""An object that can accumulate lines of Python source before execution. |
|
182 | 166 | |
|
183 | 167 | This object is designed to be fed python source line-by-line, using |
|
184 | 168 | :meth:`push`. It will return on each push whether the currently pushed |
|
185 | 169 | code could be executed already. In addition, it provides a method called |
|
186 | 170 | :meth:`push_accepts_more` that can be used to query whether more input |
|
187 | 171 | can be pushed into a single interactive block. |
|
188 | 172 | |
|
189 | 173 | This is a simple example of how an interactive terminal-based client can use |
|
190 | 174 | this tool:: |
|
191 | 175 | |
|
192 | 176 | isp = InputSplitter() |
|
193 | 177 | while isp.push_accepts_more(): |
|
194 | 178 | indent = ' '*isp.indent_spaces |
|
195 | 179 | prompt = '>>> ' + indent |
|
196 | 180 | line = indent + raw_input(prompt) |
|
197 | 181 | isp.push(line) |
|
198 | 182 | print 'Input source was:\n', isp.source_reset(), |
|
199 | 183 | """ |
|
200 | 184 | # Number of spaces of indentation computed from input that has been pushed |
|
201 | 185 | # so far. This is the attributes callers should query to get the current |
|
202 | 186 | # indentation level, in order to provide auto-indent facilities. |
|
203 | 187 | indent_spaces = 0 |
|
204 | 188 | # String, indicating the default input encoding. It is computed by default |
|
205 | 189 | # at initialization time via get_input_encoding(), but it can be reset by a |
|
206 | 190 | # client with specific knowledge of the encoding. |
|
207 | 191 | encoding = '' |
|
208 | 192 | # String where the current full source input is stored, properly encoded. |
|
209 | 193 | # Reading this attribute is the normal way of querying the currently pushed |
|
210 | 194 | # source code, that has been properly encoded. |
|
211 | 195 | source = '' |
|
212 | 196 | # Code object corresponding to the current source. It is automatically |
|
213 | 197 | # synced to the source, so it can be queried at any time to obtain the code |
|
214 | 198 | # object; it will be None if the source doesn't compile to valid Python. |
|
215 | 199 | code = None |
|
216 | 200 | |
|
217 | 201 | # Private attributes |
|
218 | 202 | |
|
219 | 203 | # List with lines of input accumulated so far |
|
220 | 204 | _buffer = None |
|
221 | 205 | # Command compiler |
|
222 | 206 | _compile = None |
|
223 | 207 | # Mark when input has changed indentation all the way back to flush-left |
|
224 | 208 | _full_dedent = False |
|
225 | 209 | # Boolean indicating whether the current block is complete |
|
226 | 210 | _is_complete = None |
|
227 | 211 | |
|
228 | 212 | def __init__(self): |
|
229 | 213 | """Create a new InputSplitter instance. |
|
230 | 214 | """ |
|
231 | 215 | self._buffer = [] |
|
232 | 216 | self._compile = codeop.CommandCompiler() |
|
233 | 217 | self.encoding = get_input_encoding() |
|
234 | 218 | |
|
235 | 219 | def reset(self): |
|
236 | 220 | """Reset the input buffer and associated state.""" |
|
237 | 221 | self.indent_spaces = 0 |
|
238 | 222 | self._buffer[:] = [] |
|
239 | 223 | self.source = '' |
|
240 | 224 | self.code = None |
|
241 | 225 | self._is_complete = False |
|
242 | 226 | self._full_dedent = False |
|
243 | 227 | |
|
244 | 228 | def source_reset(self): |
|
245 | 229 | """Return the input source and perform a full reset. |
|
246 | 230 | """ |
|
247 | 231 | out = self.source |
|
248 | 232 | self.reset() |
|
249 | 233 | return out |
|
250 | 234 | |
|
251 | 235 | def push(self, lines): |
|
252 | 236 | """Push one or more lines of input. |
|
253 | 237 | |
|
254 | 238 | This stores the given lines and returns a status code indicating |
|
255 | 239 | whether the code forms a complete Python block or not. |
|
256 | 240 | |
|
257 | 241 | Any exceptions generated in compilation are swallowed, but if an |
|
258 | 242 | exception was produced, the method returns True. |
|
259 | 243 | |
|
260 | 244 | Parameters |
|
261 | 245 | ---------- |
|
262 | 246 | lines : string |
|
263 | 247 | One or more lines of Python input. |
|
264 | 248 | |
|
265 | 249 | Returns |
|
266 | 250 | ------- |
|
267 | 251 | is_complete : boolean |
|
268 | 252 | True if the current input source (the result of the current input |
|
269 | 253 | plus prior inputs) forms a complete Python execution block. Note that |
|
270 | 254 | this value is also stored as a private attribute (``_is_complete``), so it |
|
271 | 255 | can be queried at any time. |
|
272 | 256 | """ |
|
273 | 257 | self._store(lines) |
|
274 | 258 | source = self.source |
|
275 | 259 | |
|
276 | 260 | # Before calling _compile(), reset the code object to None so that if an |
|
277 | 261 | # exception is raised in compilation, we don't mislead by having |
|
278 | 262 | # inconsistent code/source attributes. |
|
279 | 263 | self.code, self._is_complete = None, None |
|
280 | 264 | |
|
281 | 265 | # Honor termination lines properly |
|
282 | 266 | if source.endswith('\\\n'): |
|
283 | 267 | return False |
|
284 | 268 | |
|
285 | 269 | self._update_indent(lines) |
|
286 | 270 | try: |
|
287 | 271 | self.code = self._compile(source, symbol="exec") |
|
288 | 272 | # Invalid syntax can produce any of a number of different errors from |
|
289 | 273 | # inside the compiler, so we have to catch them all. Syntax errors |
|
290 | 274 | # immediately produce a 'ready' block, so the invalid Python can be |
|
291 | 275 | # sent to the kernel for evaluation with possible ipython |
|
292 | 276 | # special-syntax conversion. |
|
293 | 277 | except (SyntaxError, OverflowError, ValueError, TypeError, |
|
294 | 278 | MemoryError): |
|
295 | 279 | self._is_complete = True |
|
296 | 280 | else: |
|
297 | 281 | # Compilation didn't produce any exceptions (though it may not have |
|
298 | 282 | # given a complete code object) |
|
299 | 283 | self._is_complete = self.code is not None |
|
300 | 284 | |
|
301 | 285 | return self._is_complete |
|
302 | 286 | |
|
303 | 287 | def push_accepts_more(self): |
|
304 | 288 | """Return whether a block of interactive input can accept more input. |
|
305 | 289 | |
|
306 | 290 | This method is meant to be used by line-oriented frontends, who need to |
|
307 | 291 | guess whether a block is complete or not based solely on prior and |
|
308 | 292 | current input lines. The InputSplitter considers it has a complete |
|
309 | 293 | interactive block and will not accept more input when either: |
|
310 | 294 | |
|
311 | 295 | * A SyntaxError is raised |
|
312 | 296 | |
|
313 | 297 | * The code is complete and consists of a single line or a single |
|
314 | 298 | non-compound statement |
|
315 | 299 | |
|
316 | 300 | * The code is complete and has a blank line at the end |
|
317 | 301 | |
|
318 | 302 | If the current input produces a syntax error, this method immediately |
|
319 | 303 | returns False but does *not* raise the syntax error exception, as |
|
320 | 304 | typically clients will want to send invalid syntax to an execution |
|
321 | 305 | backend which might convert the invalid syntax into valid Python via |
|
322 | 306 | one of the dynamic IPython mechanisms. |
|
323 | 307 | """ |
|
324 | 308 | |
|
325 | 309 | # With incomplete input, unconditionally accept more |
|
326 | 310 | # A syntax error also sets _is_complete to True - see push() |
|
327 | 311 | if not self._is_complete: |
|
328 | 312 | #print("Not complete") # debug |
|
329 | 313 | return True |
|
330 | 314 | |
|
331 | 315 | # The user can make any (complete) input execute by leaving a blank line |
|
332 | 316 | last_line = self.source.splitlines()[-1] |
|
333 | 317 | if (not last_line) or last_line.isspace(): |
|
334 | 318 | #print("Blank line") # debug |
|
335 | 319 | return False |
|
336 | 320 | |
|
337 | 321 | # If there's just a single line or AST node, and we're flush left, as is |
|
338 | 322 | # the case after a simple statement such as 'a=1', we want to execute it |
|
339 | 323 | # straight away. |
|
340 | 324 | if self.indent_spaces==0: |
|
341 | 325 | if len(self.source.splitlines()) <= 1: |
|
342 | 326 | return False |
|
343 | 327 | |
|
344 | 328 | try: |
|
345 | 329 | code_ast = ast.parse(u''.join(self._buffer)) |
|
346 | 330 | except Exception: |
|
347 | 331 | #print("Can't parse AST") # debug |
|
348 | 332 | return False |
|
349 | 333 | else: |
|
350 | 334 | if len(code_ast.body) == 1 and \ |
|
351 | 335 | not hasattr(code_ast.body[0], 'body'): |
|
352 | 336 | #print("Simple statement") # debug |
|
353 | 337 | return False |
|
354 | 338 | |
|
355 | 339 | # General fallback - accept more code |
|
356 | 340 | return True |
|
357 | 341 | |
|
358 | 342 | #------------------------------------------------------------------------ |
|
359 | 343 | # Private interface |
|
360 | 344 | #------------------------------------------------------------------------ |
|
361 | 345 | |
|
362 | 346 | def _find_indent(self, line): |
|
363 | 347 | """Compute the new indentation level for a single line. |
|
364 | 348 | |
|
365 | 349 | Parameters |
|
366 | 350 | ---------- |
|
367 | 351 | line : str |
|
368 | 352 | A single new line of non-whitespace, non-comment Python input. |
|
369 | 353 | |
|
370 | 354 | Returns |
|
371 | 355 | ------- |
|
372 | 356 | indent_spaces : int |
|
373 | 357 | New value for the indent level (it may be equal to self.indent_spaces |
|
374 | 358 | if indentation doesn't change. |
|
375 | 359 | |
|
376 | 360 | full_dedent : boolean |
|
377 | 361 | Whether the new line causes a full flush-left dedent. |
|
378 | 362 | """ |
|
379 | 363 | indent_spaces = self.indent_spaces |
|
380 | 364 | full_dedent = self._full_dedent |
|
381 | 365 | |
|
382 | 366 | inisp = num_ini_spaces(line) |
|
383 | 367 | if inisp < indent_spaces: |
|
384 | 368 | indent_spaces = inisp |
|
385 | 369 | if indent_spaces <= 0: |
|
386 | 370 | #print 'Full dedent in text',self.source # dbg |
|
387 | 371 | full_dedent = True |
|
388 | 372 | |
|
389 | 373 | if line.rstrip()[-1] == ':': |
|
390 | 374 | indent_spaces += 4 |
|
391 | 375 | elif dedent_re.match(line): |
|
392 | 376 | indent_spaces -= 4 |
|
393 | 377 | if indent_spaces <= 0: |
|
394 | 378 | full_dedent = True |
|
395 | 379 | |
|
396 | 380 | # Safety |
|
397 | 381 | if indent_spaces < 0: |
|
398 | 382 | indent_spaces = 0 |
|
399 | 383 | #print 'safety' # dbg |
|
400 | 384 | |
|
401 | 385 | return indent_spaces, full_dedent |
|
402 | 386 | |
|
403 | 387 | def _update_indent(self, lines): |
|
404 | 388 | for line in remove_comments(lines).splitlines(): |
|
405 | 389 | if line and not line.isspace(): |
|
406 | 390 | self.indent_spaces, self._full_dedent = self._find_indent(line) |
|
407 | 391 | |
|
408 | 392 | def _store(self, lines, buffer=None, store='source'): |
|
409 | 393 | """Store one or more lines of input. |
|
410 | 394 | |
|
411 | 395 | If input lines are not newline-terminated, a newline is automatically |
|
412 | 396 | appended.""" |
|
413 | 397 | |
|
414 | 398 | if buffer is None: |
|
415 | 399 | buffer = self._buffer |
|
416 | 400 | |
|
417 | 401 | if lines.endswith('\n'): |
|
418 | 402 | buffer.append(lines) |
|
419 | 403 | else: |
|
420 | 404 | buffer.append(lines+'\n') |
|
421 | 405 | setattr(self, store, self._set_source(buffer)) |
|
422 | 406 | |
|
423 | 407 | def _set_source(self, buffer): |
|
424 | 408 | return u''.join(buffer) |
|
425 | 409 | |
|
426 | 410 | |
|
427 | 411 | class IPythonInputSplitter(InputSplitter): |
|
428 | 412 | """An input splitter that recognizes all of IPython's special syntax.""" |
|
429 | 413 | |
|
430 | 414 | # String with raw, untransformed input. |
|
431 | 415 | source_raw = '' |
|
432 | 416 | |
|
433 | 417 | # Flag to track when a transformer has stored input that it hasn't given |
|
434 | 418 | # back yet. |
|
435 | 419 | transformer_accumulating = False |
|
436 | 420 | |
|
437 | 421 | # Flag to track when assemble_python_lines has stored input that it hasn't |
|
438 | 422 | # given back yet. |
|
439 | 423 | within_python_line = False |
|
440 | 424 | |
|
441 | 425 | # Private attributes |
|
442 | 426 | |
|
443 | 427 | # List with lines of raw input accumulated so far. |
|
444 | 428 | _buffer_raw = None |
|
445 | 429 | |
|
446 | 430 | def __init__(self, line_input_checker=True, physical_line_transforms=None, |
|
447 | 431 | logical_line_transforms=None, python_line_transforms=None): |
|
448 | 432 | super(IPythonInputSplitter, self).__init__() |
|
449 | 433 | self._buffer_raw = [] |
|
450 | 434 | self._validate = True |
|
451 | 435 | |
|
452 | 436 | if physical_line_transforms is not None: |
|
453 | 437 | self.physical_line_transforms = physical_line_transforms |
|
454 | 438 | else: |
|
455 | 439 | self.physical_line_transforms = [ |
|
456 | 440 | leading_indent(), |
|
457 | 441 | classic_prompt(), |
|
458 | 442 | ipy_prompt(), |
|
459 | 443 | strip_encoding_cookie(), |
|
460 | 444 | cellmagic(end_on_blank_line=line_input_checker), |
|
461 | 445 | ] |
|
462 | 446 | |
|
463 | 447 | self.assemble_logical_lines = assemble_logical_lines() |
|
464 | 448 | if logical_line_transforms is not None: |
|
465 | 449 | self.logical_line_transforms = logical_line_transforms |
|
466 | 450 | else: |
|
467 | 451 | self.logical_line_transforms = [ |
|
468 | 452 | help_end(), |
|
469 | 453 | escaped_commands(), |
|
470 | 454 | assign_from_magic(), |
|
471 | 455 | assign_from_system(), |
|
472 | 456 | ] |
|
473 | 457 | |
|
474 | 458 | self.assemble_python_lines = assemble_python_lines() |
|
475 | 459 | if python_line_transforms is not None: |
|
476 | 460 | self.python_line_transforms = python_line_transforms |
|
477 | 461 | else: |
|
478 | 462 | # We don't use any of these at present |
|
479 | 463 | self.python_line_transforms = [] |
|
480 | 464 | |
|
481 | 465 | @property |
|
482 | 466 | def transforms(self): |
|
483 | 467 | "Quick access to all transformers." |
|
484 | 468 | return self.physical_line_transforms + \ |
|
485 | 469 | [self.assemble_logical_lines] + self.logical_line_transforms + \ |
|
486 | 470 | [self.assemble_python_lines] + self.python_line_transforms |
|
487 | 471 | |
|
488 | 472 | @property |
|
489 | 473 | def transforms_in_use(self): |
|
490 | 474 | """Transformers, excluding logical line transformers if we're in a |
|
491 | 475 | Python line.""" |
|
492 | 476 | t = self.physical_line_transforms[:] |
|
493 | 477 | if not self.within_python_line: |
|
494 | 478 | t += [self.assemble_logical_lines] + self.logical_line_transforms |
|
495 | 479 | return t + [self.assemble_python_lines] + self.python_line_transforms |
|
496 | 480 | |
|
497 | 481 | def reset(self): |
|
498 | 482 | """Reset the input buffer and associated state.""" |
|
499 | 483 | super(IPythonInputSplitter, self).reset() |
|
500 | 484 | self._buffer_raw[:] = [] |
|
501 | 485 | self.source_raw = '' |
|
502 | 486 | self.transformer_accumulating = False |
|
503 | 487 | self.within_python_line = False |
|
504 | 488 | |
|
505 | 489 | for t in self.transforms: |
|
506 | 490 | try: |
|
507 | 491 | t.reset() |
|
508 | 492 | except SyntaxError: |
|
509 | 493 | # Nothing that calls reset() expects to handle transformer |
|
510 | 494 | # errors |
|
511 | 495 | pass |
|
512 | 496 | |
|
513 | 497 | def flush_transformers(self): |
|
514 | 498 | def _flush(transform, out): |
|
515 | if out is not None: | |
|
516 | tmp = transform.push(out) | |
|
517 | return tmp or transform.reset() or None | |
|
518 |
|
|
|
519 | return transform.reset() or None | |
|
499 | """yield transformed lines | |
|
500 | ||
|
501 | always strings, never None | |
|
502 | ||
|
503 | transform: the current transform | |
|
504 | out: an iterable of previously transformed inputs. | |
|
505 | Each may be multiline, which will be passed | |
|
506 | one line at a time to transform. | |
|
507 | """ | |
|
508 | anything = False | |
|
509 | for out in out: | |
|
510 | anything = True | |
|
511 | tmp = None | |
|
512 | for line in out.splitlines(): | |
|
513 | # push one line at a time | |
|
514 | tmp = transform.push(line) | |
|
515 | if tmp is not None: | |
|
516 | yield tmp | |
|
517 | if tmp is None: | |
|
518 | # transformer is still consuming, reset | |
|
519 | tmp = transform.reset() | |
|
520 | if tmp is not None: | |
|
521 | yield tmp | |
|
522 | if not anything: | |
|
523 | # nothing was pushed, reset | |
|
524 | tmp = transform.reset() | |
|
525 | if tmp is not None: | |
|
526 | yield tmp | |
|
520 | 527 | |
|
521 |
out = |
|
|
528 | out = [] | |
|
522 | 529 | for t in self.transforms_in_use: |
|
523 | 530 | out = _flush(t, out) |
|
524 | 531 | |
|
525 |
|
|
|
526 | self._store(out) | |
|
532 | out = list(out) | |
|
533 | if out: | |
|
534 | self._store('\n'.join(out)) | |
|
527 | 535 | |
|
528 | 536 | def raw_reset(self): |
|
529 | 537 | """Return raw input only and perform a full reset. |
|
530 | 538 | """ |
|
531 | 539 | out = self.source_raw |
|
532 | 540 | self.reset() |
|
533 | 541 | return out |
|
534 | 542 | |
|
535 | 543 | def source_reset(self): |
|
536 | 544 | try: |
|
537 | 545 | self.flush_transformers() |
|
538 | 546 | return self.source |
|
539 | 547 | finally: |
|
540 | 548 | self.reset() |
|
541 | 549 | |
|
542 | 550 | def push_accepts_more(self): |
|
543 | 551 | if self.transformer_accumulating: |
|
544 | 552 | return True |
|
545 | 553 | else: |
|
546 | 554 | return super(IPythonInputSplitter, self).push_accepts_more() |
|
547 | 555 | |
|
548 | 556 | def transform_cell(self, cell): |
|
549 | 557 | """Process and translate a cell of input. |
|
550 | 558 | """ |
|
551 | 559 | self.reset() |
|
552 | 560 | try: |
|
553 | 561 | self.push(cell) |
|
554 | 562 | self.flush_transformers() |
|
555 | 563 | return self.source |
|
556 | 564 | finally: |
|
557 | 565 | self.reset() |
|
558 | 566 | |
|
559 | 567 | def push(self, lines): |
|
560 | 568 | """Push one or more lines of IPython input. |
|
561 | 569 | |
|
562 | 570 | This stores the given lines and returns a status code indicating |
|
563 | 571 | whether the code forms a complete Python block or not, after processing |
|
564 | 572 | all input lines for special IPython syntax. |
|
565 | 573 | |
|
566 | 574 | Any exceptions generated in compilation are swallowed, but if an |
|
567 | 575 | exception was produced, the method returns True. |
|
568 | 576 | |
|
569 | 577 | Parameters |
|
570 | 578 | ---------- |
|
571 | 579 | lines : string |
|
572 | 580 | One or more lines of Python input. |
|
573 | 581 | |
|
574 | 582 | Returns |
|
575 | 583 | ------- |
|
576 | 584 | is_complete : boolean |
|
577 | 585 | True if the current input source (the result of the current input |
|
578 | 586 | plus prior inputs) forms a complete Python execution block. Note that |
|
579 | 587 | this value is also stored as a private attribute (_is_complete), so it |
|
580 | 588 | can be queried at any time. |
|
581 | 589 | """ |
|
582 | 590 | |
|
583 | 591 | # We must ensure all input is pure unicode |
|
584 | 592 | lines = cast_unicode(lines, self.encoding) |
|
585 | 593 | |
|
586 | 594 | # ''.splitlines() --> [], but we need to push the empty line to transformers |
|
587 | 595 | lines_list = lines.splitlines() |
|
588 | 596 | if not lines_list: |
|
589 | 597 | lines_list = [''] |
|
590 | 598 | |
|
591 | 599 | # Store raw source before applying any transformations to it. Note |
|
592 | 600 | # that this must be done *after* the reset() call that would otherwise |
|
593 | 601 | # flush the buffer. |
|
594 | 602 | self._store(lines, self._buffer_raw, 'source_raw') |
|
595 | 603 | |
|
596 | 604 | for line in lines_list: |
|
597 | 605 | out = self.push_line(line) |
|
598 | 606 | |
|
599 | 607 | return out |
|
600 | 608 | |
|
601 | 609 | def push_line(self, line): |
|
602 | 610 | buf = self._buffer |
|
603 | 611 | |
|
604 | 612 | def _accumulating(dbg): |
|
605 | 613 | #print(dbg) |
|
606 | 614 | self.transformer_accumulating = True |
|
607 | 615 | return False |
|
608 | 616 | |
|
609 | 617 | for transformer in self.physical_line_transforms: |
|
610 | 618 | line = transformer.push(line) |
|
611 | 619 | if line is None: |
|
612 | 620 | return _accumulating(transformer) |
|
613 | 621 | |
|
614 | 622 | if not self.within_python_line: |
|
615 | 623 | line = self.assemble_logical_lines.push(line) |
|
616 | 624 | if line is None: |
|
617 | 625 | return _accumulating('acc logical line') |
|
618 | 626 | |
|
619 | 627 | for transformer in self.logical_line_transforms: |
|
620 | 628 | line = transformer.push(line) |
|
621 | 629 | if line is None: |
|
622 | 630 | return _accumulating(transformer) |
|
623 | 631 | |
|
624 | 632 | line = self.assemble_python_lines.push(line) |
|
625 | 633 | if line is None: |
|
626 | 634 | self.within_python_line = True |
|
627 | 635 | return _accumulating('acc python line') |
|
628 | 636 | else: |
|
629 | 637 | self.within_python_line = False |
|
630 | 638 | |
|
631 | 639 | for transformer in self.python_line_transforms: |
|
632 | 640 | line = transformer.push(line) |
|
633 | 641 | if line is None: |
|
634 | 642 | return _accumulating(transformer) |
|
635 | 643 | |
|
636 | 644 | #print("transformers clear") #debug |
|
637 | 645 | self.transformer_accumulating = False |
|
638 | 646 | return super(IPythonInputSplitter, self).push(line) |
@@ -1,585 +1,597 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | """Tests for the inputsplitter module. | |
|
2 | """Tests for the inputsplitter module.""" | |
|
3 | 3 | |
|
4 | Authors | |
|
5 | ------- | |
|
6 | * Fernando Perez | |
|
7 | * Robert Kern | |
|
8 | """ | |
|
9 | 4 | from __future__ import print_function |
|
10 | #----------------------------------------------------------------------------- | |
|
11 | # Copyright (C) 2010-2011 The IPython Development Team | |
|
12 | # | |
|
13 | # Distributed under the terms of the BSD License. The full license is in | |
|
14 | # the file COPYING, distributed as part of this software. | |
|
15 | #----------------------------------------------------------------------------- | |
|
16 | 5 | |
|
17 | #----------------------------------------------------------------------------- | |
|
18 | # Imports | |
|
19 | #----------------------------------------------------------------------------- | |
|
20 | # stdlib | |
|
6 | # Copyright (c) IPython Development Team. | |
|
7 | # Distributed under the terms of the Modified BSD License. | |
|
8 | ||
|
21 | 9 | import unittest |
|
22 | 10 | import sys |
|
23 | 11 | |
|
24 | # Third party | |
|
25 | 12 | import nose.tools as nt |
|
26 | 13 | |
|
27 | # Our own | |
|
28 | 14 | from IPython.core import inputsplitter as isp |
|
15 | from IPython.core.inputtransformer import InputTransformer | |
|
29 | 16 | from IPython.core.tests.test_inputtransformer import syntax, syntax_ml |
|
30 | 17 | from IPython.testing import tools as tt |
|
31 | 18 | from IPython.utils import py3compat |
|
32 | 19 | from IPython.utils.py3compat import string_types, input |
|
33 | 20 | |
|
34 | 21 | #----------------------------------------------------------------------------- |
|
35 | 22 | # Semi-complete examples (also used as tests) |
|
36 | 23 | #----------------------------------------------------------------------------- |
|
37 | 24 | |
|
38 | 25 | # Note: at the bottom, there's a slightly more complete version of this that |
|
39 | 26 | # can be useful during development of code here. |
|
40 | 27 | |
|
41 | 28 | def mini_interactive_loop(input_func): |
|
42 | 29 | """Minimal example of the logic of an interactive interpreter loop. |
|
43 | 30 | |
|
44 | 31 | This serves as an example, and it is used by the test system with a fake |
|
45 | 32 | raw_input that simulates interactive input.""" |
|
46 | 33 | |
|
47 | 34 | from IPython.core.inputsplitter import InputSplitter |
|
48 | 35 | |
|
49 | 36 | isp = InputSplitter() |
|
50 | 37 | # In practice, this input loop would be wrapped in an outside loop to read |
|
51 | 38 | # input indefinitely, until some exit/quit command was issued. Here we |
|
52 | 39 | # only illustrate the basic inner loop. |
|
53 | 40 | while isp.push_accepts_more(): |
|
54 | 41 | indent = ' '*isp.indent_spaces |
|
55 | 42 | prompt = '>>> ' + indent |
|
56 | 43 | line = indent + input_func(prompt) |
|
57 | 44 | isp.push(line) |
|
58 | 45 | |
|
59 | 46 | # Here we just return input so we can use it in a test suite, but a real |
|
60 | 47 | # interpreter would instead send it for execution somewhere. |
|
61 | 48 | src = isp.source_reset() |
|
62 | 49 | #print 'Input source was:\n', src # dbg |
|
63 | 50 | return src |
|
64 | 51 | |
|
65 | 52 | #----------------------------------------------------------------------------- |
|
66 | 53 | # Test utilities, just for local use |
|
67 | 54 | #----------------------------------------------------------------------------- |
|
68 | 55 | |
|
69 | 56 | def assemble(block): |
|
70 | 57 | """Assemble a block into multi-line sub-blocks.""" |
|
71 | 58 | return ['\n'.join(sub_block)+'\n' for sub_block in block] |
|
72 | 59 | |
|
73 | 60 | |
|
74 | 61 | def pseudo_input(lines): |
|
75 | 62 | """Return a function that acts like raw_input but feeds the input list.""" |
|
76 | 63 | ilines = iter(lines) |
|
77 | 64 | def raw_in(prompt): |
|
78 | 65 | try: |
|
79 | 66 | return next(ilines) |
|
80 | 67 | except StopIteration: |
|
81 | 68 | return '' |
|
82 | 69 | return raw_in |
|
83 | 70 | |
|
84 | 71 | #----------------------------------------------------------------------------- |
|
85 | 72 | # Tests |
|
86 | 73 | #----------------------------------------------------------------------------- |
|
87 | 74 | def test_spaces(): |
|
88 | 75 | tests = [('', 0), |
|
89 | 76 | (' ', 1), |
|
90 | 77 | ('\n', 0), |
|
91 | 78 | (' \n', 1), |
|
92 | 79 | ('x', 0), |
|
93 | 80 | (' x', 1), |
|
94 | 81 | (' x',2), |
|
95 | 82 | (' x',4), |
|
96 | 83 | # Note: tabs are counted as a single whitespace! |
|
97 | 84 | ('\tx', 1), |
|
98 | 85 | ('\t x', 2), |
|
99 | 86 | ] |
|
100 | 87 | tt.check_pairs(isp.num_ini_spaces, tests) |
|
101 | 88 | |
|
102 | 89 | |
|
103 | 90 | def test_remove_comments(): |
|
104 | 91 | tests = [('text', 'text'), |
|
105 | 92 | ('text # comment', 'text '), |
|
106 | 93 | ('text # comment\n', 'text \n'), |
|
107 | 94 | ('text # comment \n', 'text \n'), |
|
108 | 95 | ('line # c \nline\n','line \nline\n'), |
|
109 | 96 | ('line # c \nline#c2 \nline\nline #c\n\n', |
|
110 | 97 | 'line \nline\nline\nline \n\n'), |
|
111 | 98 | ] |
|
112 | 99 | tt.check_pairs(isp.remove_comments, tests) |
|
113 | 100 | |
|
114 | 101 | |
|
115 | 102 | def test_get_input_encoding(): |
|
116 | 103 | encoding = isp.get_input_encoding() |
|
117 | 104 | nt.assert_true(isinstance(encoding, string_types)) |
|
118 | 105 | # simple-minded check that at least encoding a simple string works with the |
|
119 | 106 | # encoding we got. |
|
120 | 107 | nt.assert_equal(u'test'.encode(encoding), b'test') |
|
121 | 108 | |
|
122 | 109 | |
|
123 | 110 | class NoInputEncodingTestCase(unittest.TestCase): |
|
124 | 111 | def setUp(self): |
|
125 | 112 | self.old_stdin = sys.stdin |
|
126 | 113 | class X: pass |
|
127 | 114 | fake_stdin = X() |
|
128 | 115 | sys.stdin = fake_stdin |
|
129 | 116 | |
|
130 | 117 | def test(self): |
|
131 | 118 | # Verify that if sys.stdin has no 'encoding' attribute we do the right |
|
132 | 119 | # thing |
|
133 | 120 | enc = isp.get_input_encoding() |
|
134 | 121 | self.assertEqual(enc, 'ascii') |
|
135 | 122 | |
|
136 | 123 | def tearDown(self): |
|
137 | 124 | sys.stdin = self.old_stdin |
|
138 | 125 | |
|
139 | 126 | |
|
140 | 127 | class InputSplitterTestCase(unittest.TestCase): |
|
141 | 128 | def setUp(self): |
|
142 | 129 | self.isp = isp.InputSplitter() |
|
143 | 130 | |
|
144 | 131 | def test_reset(self): |
|
145 | 132 | isp = self.isp |
|
146 | 133 | isp.push('x=1') |
|
147 | 134 | isp.reset() |
|
148 | 135 | self.assertEqual(isp._buffer, []) |
|
149 | 136 | self.assertEqual(isp.indent_spaces, 0) |
|
150 | 137 | self.assertEqual(isp.source, '') |
|
151 | 138 | self.assertEqual(isp.code, None) |
|
152 | 139 | self.assertEqual(isp._is_complete, False) |
|
153 | 140 | |
|
154 | 141 | def test_source(self): |
|
155 | 142 | self.isp._store('1') |
|
156 | 143 | self.isp._store('2') |
|
157 | 144 | self.assertEqual(self.isp.source, '1\n2\n') |
|
158 | 145 | self.assertTrue(len(self.isp._buffer)>0) |
|
159 | 146 | self.assertEqual(self.isp.source_reset(), '1\n2\n') |
|
160 | 147 | self.assertEqual(self.isp._buffer, []) |
|
161 | 148 | self.assertEqual(self.isp.source, '') |
|
162 | 149 | |
|
163 | 150 | def test_indent(self): |
|
164 | 151 | isp = self.isp # shorthand |
|
165 | 152 | isp.push('x=1') |
|
166 | 153 | self.assertEqual(isp.indent_spaces, 0) |
|
167 | 154 | isp.push('if 1:\n x=1') |
|
168 | 155 | self.assertEqual(isp.indent_spaces, 4) |
|
169 | 156 | isp.push('y=2\n') |
|
170 | 157 | self.assertEqual(isp.indent_spaces, 0) |
|
171 | 158 | |
|
172 | 159 | def test_indent2(self): |
|
173 | 160 | isp = self.isp |
|
174 | 161 | isp.push('if 1:') |
|
175 | 162 | self.assertEqual(isp.indent_spaces, 4) |
|
176 | 163 | isp.push(' x=1') |
|
177 | 164 | self.assertEqual(isp.indent_spaces, 4) |
|
178 | 165 | # Blank lines shouldn't change the indent level |
|
179 | 166 | isp.push(' '*2) |
|
180 | 167 | self.assertEqual(isp.indent_spaces, 4) |
|
181 | 168 | |
|
182 | 169 | def test_indent3(self): |
|
183 | 170 | isp = self.isp |
|
184 | 171 | # When a multiline statement contains parens or multiline strings, we |
|
185 | 172 | # shouldn't get confused. |
|
186 | 173 | isp.push("if 1:") |
|
187 | 174 | isp.push(" x = (1+\n 2)") |
|
188 | 175 | self.assertEqual(isp.indent_spaces, 4) |
|
189 | 176 | |
|
190 | 177 | def test_indent4(self): |
|
191 | 178 | isp = self.isp |
|
192 | 179 | # whitespace after ':' should not screw up indent level |
|
193 | 180 | isp.push('if 1: \n x=1') |
|
194 | 181 | self.assertEqual(isp.indent_spaces, 4) |
|
195 | 182 | isp.push('y=2\n') |
|
196 | 183 | self.assertEqual(isp.indent_spaces, 0) |
|
197 | 184 | isp.push('if 1:\t\n x=1') |
|
198 | 185 | self.assertEqual(isp.indent_spaces, 4) |
|
199 | 186 | isp.push('y=2\n') |
|
200 | 187 | self.assertEqual(isp.indent_spaces, 0) |
|
201 | 188 | |
|
202 | 189 | def test_dedent_pass(self): |
|
203 | 190 | isp = self.isp # shorthand |
|
204 | 191 | # should NOT cause dedent |
|
205 | 192 | isp.push('if 1:\n passes = 5') |
|
206 | 193 | self.assertEqual(isp.indent_spaces, 4) |
|
207 | 194 | isp.push('if 1:\n pass') |
|
208 | 195 | self.assertEqual(isp.indent_spaces, 0) |
|
209 | 196 | isp.push('if 1:\n pass ') |
|
210 | 197 | self.assertEqual(isp.indent_spaces, 0) |
|
211 | 198 | |
|
212 | 199 | def test_dedent_break(self): |
|
213 | 200 | isp = self.isp # shorthand |
|
214 | 201 | # should NOT cause dedent |
|
215 | 202 | isp.push('while 1:\n breaks = 5') |
|
216 | 203 | self.assertEqual(isp.indent_spaces, 4) |
|
217 | 204 | isp.push('while 1:\n break') |
|
218 | 205 | self.assertEqual(isp.indent_spaces, 0) |
|
219 | 206 | isp.push('while 1:\n break ') |
|
220 | 207 | self.assertEqual(isp.indent_spaces, 0) |
|
221 | 208 | |
|
222 | 209 | def test_dedent_continue(self): |
|
223 | 210 | isp = self.isp # shorthand |
|
224 | 211 | # should NOT cause dedent |
|
225 | 212 | isp.push('while 1:\n continues = 5') |
|
226 | 213 | self.assertEqual(isp.indent_spaces, 4) |
|
227 | 214 | isp.push('while 1:\n continue') |
|
228 | 215 | self.assertEqual(isp.indent_spaces, 0) |
|
229 | 216 | isp.push('while 1:\n continue ') |
|
230 | 217 | self.assertEqual(isp.indent_spaces, 0) |
|
231 | 218 | |
|
232 | 219 | def test_dedent_raise(self): |
|
233 | 220 | isp = self.isp # shorthand |
|
234 | 221 | # should NOT cause dedent |
|
235 | 222 | isp.push('if 1:\n raised = 4') |
|
236 | 223 | self.assertEqual(isp.indent_spaces, 4) |
|
237 | 224 | isp.push('if 1:\n raise TypeError()') |
|
238 | 225 | self.assertEqual(isp.indent_spaces, 0) |
|
239 | 226 | isp.push('if 1:\n raise') |
|
240 | 227 | self.assertEqual(isp.indent_spaces, 0) |
|
241 | 228 | isp.push('if 1:\n raise ') |
|
242 | 229 | self.assertEqual(isp.indent_spaces, 0) |
|
243 | 230 | |
|
244 | 231 | def test_dedent_return(self): |
|
245 | 232 | isp = self.isp # shorthand |
|
246 | 233 | # should NOT cause dedent |
|
247 | 234 | isp.push('if 1:\n returning = 4') |
|
248 | 235 | self.assertEqual(isp.indent_spaces, 4) |
|
249 | 236 | isp.push('if 1:\n return 5 + 493') |
|
250 | 237 | self.assertEqual(isp.indent_spaces, 0) |
|
251 | 238 | isp.push('if 1:\n return') |
|
252 | 239 | self.assertEqual(isp.indent_spaces, 0) |
|
253 | 240 | isp.push('if 1:\n return ') |
|
254 | 241 | self.assertEqual(isp.indent_spaces, 0) |
|
255 | 242 | isp.push('if 1:\n return(0)') |
|
256 | 243 | self.assertEqual(isp.indent_spaces, 0) |
|
257 | 244 | |
|
258 | 245 | def test_push(self): |
|
259 | 246 | isp = self.isp |
|
260 | 247 | self.assertTrue(isp.push('x=1')) |
|
261 | 248 | |
|
262 | 249 | def test_push2(self): |
|
263 | 250 | isp = self.isp |
|
264 | 251 | self.assertFalse(isp.push('if 1:')) |
|
265 | 252 | for line in [' x=1', '# a comment', ' y=2']: |
|
266 | 253 | print(line) |
|
267 | 254 | self.assertTrue(isp.push(line)) |
|
268 | 255 | |
|
269 | 256 | def test_push3(self): |
|
270 | 257 | isp = self.isp |
|
271 | 258 | isp.push('if True:') |
|
272 | 259 | isp.push(' a = 1') |
|
273 | 260 | self.assertFalse(isp.push('b = [1,')) |
|
274 | 261 | |
|
275 | 262 | def test_push_accepts_more(self): |
|
276 | 263 | isp = self.isp |
|
277 | 264 | isp.push('x=1') |
|
278 | 265 | self.assertFalse(isp.push_accepts_more()) |
|
279 | 266 | |
|
280 | 267 | def test_push_accepts_more2(self): |
|
281 | 268 | isp = self.isp |
|
282 | 269 | isp.push('if 1:') |
|
283 | 270 | self.assertTrue(isp.push_accepts_more()) |
|
284 | 271 | isp.push(' x=1') |
|
285 | 272 | self.assertTrue(isp.push_accepts_more()) |
|
286 | 273 | isp.push('') |
|
287 | 274 | self.assertFalse(isp.push_accepts_more()) |
|
288 | 275 | |
|
289 | 276 | def test_push_accepts_more3(self): |
|
290 | 277 | isp = self.isp |
|
291 | 278 | isp.push("x = (2+\n3)") |
|
292 | 279 | self.assertFalse(isp.push_accepts_more()) |
|
293 | 280 | |
|
294 | 281 | def test_push_accepts_more4(self): |
|
295 | 282 | isp = self.isp |
|
296 | 283 | # When a multiline statement contains parens or multiline strings, we |
|
297 | 284 | # shouldn't get confused. |
|
298 | 285 | # FIXME: we should be able to better handle de-dents in statements like |
|
299 | 286 | # multiline strings and multiline expressions (continued with \ or |
|
300 | 287 | # parens). Right now we aren't handling the indentation tracking quite |
|
301 | 288 | # correctly with this, though in practice it may not be too much of a |
|
302 | 289 | # problem. We'll need to see. |
|
303 | 290 | isp.push("if 1:") |
|
304 | 291 | isp.push(" x = (2+") |
|
305 | 292 | isp.push(" 3)") |
|
306 | 293 | self.assertTrue(isp.push_accepts_more()) |
|
307 | 294 | isp.push(" y = 3") |
|
308 | 295 | self.assertTrue(isp.push_accepts_more()) |
|
309 | 296 | isp.push('') |
|
310 | 297 | self.assertFalse(isp.push_accepts_more()) |
|
311 | 298 | |
|
312 | 299 | def test_push_accepts_more5(self): |
|
313 | 300 | isp = self.isp |
|
314 | 301 | isp.push('try:') |
|
315 | 302 | isp.push(' a = 5') |
|
316 | 303 | isp.push('except:') |
|
317 | 304 | isp.push(' raise') |
|
318 | 305 | # We want to be able to add an else: block at this point, so it should |
|
319 | 306 | # wait for a blank line. |
|
320 | 307 | self.assertTrue(isp.push_accepts_more()) |
|
321 | 308 | |
|
322 | 309 | def test_continuation(self): |
|
323 | 310 | isp = self.isp |
|
324 | 311 | isp.push("import os, \\") |
|
325 | 312 | self.assertTrue(isp.push_accepts_more()) |
|
326 | 313 | isp.push("sys") |
|
327 | 314 | self.assertFalse(isp.push_accepts_more()) |
|
328 | 315 | |
|
329 | 316 | def test_syntax_error(self): |
|
330 | 317 | isp = self.isp |
|
331 | 318 | # Syntax errors immediately produce a 'ready' block, so the invalid |
|
332 | 319 | # Python can be sent to the kernel for evaluation with possible ipython |
|
333 | 320 | # special-syntax conversion. |
|
334 | 321 | isp.push('run foo') |
|
335 | 322 | self.assertFalse(isp.push_accepts_more()) |
|
336 | 323 | |
|
337 | 324 | def test_unicode(self): |
|
338 | 325 | self.isp.push(u"PΓ©rez") |
|
339 | 326 | self.isp.push(u'\xc3\xa9') |
|
340 | 327 | self.isp.push(u"u'\xc3\xa9'") |
|
341 | 328 | |
|
342 | 329 | def test_line_continuation(self): |
|
343 | 330 | """ Test issue #2108.""" |
|
344 | 331 | isp = self.isp |
|
345 | 332 | # A blank line after a line continuation should not accept more |
|
346 | 333 | isp.push("1 \\\n\n") |
|
347 | 334 | self.assertFalse(isp.push_accepts_more()) |
|
348 | 335 | # Whitespace after a \ is a SyntaxError. The only way to test that |
|
349 | 336 | # here is to test that push doesn't accept more (as with |
|
350 | 337 | # test_syntax_error() above). |
|
351 | 338 | isp.push(r"1 \ ") |
|
352 | 339 | self.assertFalse(isp.push_accepts_more()) |
|
353 | 340 | # Even if the line is continuable (c.f. the regular Python |
|
354 | 341 | # interpreter) |
|
355 | 342 | isp.push(r"(1 \ ") |
|
356 | 343 | self.assertFalse(isp.push_accepts_more()) |
|
357 | 344 | |
|
358 | 345 | class InteractiveLoopTestCase(unittest.TestCase): |
|
359 | 346 | """Tests for an interactive loop like a python shell. |
|
360 | 347 | """ |
|
361 | 348 | def check_ns(self, lines, ns): |
|
362 | 349 | """Validate that the given input lines produce the resulting namespace. |
|
363 | 350 | |
|
364 | 351 | Note: the input lines are given exactly as they would be typed in an |
|
365 | 352 | auto-indenting environment, as mini_interactive_loop above already does |
|
366 | 353 | auto-indenting and prepends spaces to the input. |
|
367 | 354 | """ |
|
368 | 355 | src = mini_interactive_loop(pseudo_input(lines)) |
|
369 | 356 | test_ns = {} |
|
370 | 357 | exec(src, test_ns) |
|
371 | 358 | # We can't check that the provided ns is identical to the test_ns, |
|
372 | 359 | # because Python fills test_ns with extra keys (copyright, etc). But |
|
373 | 360 | # we can check that the given dict is *contained* in test_ns |
|
374 | 361 | for k,v in ns.items(): |
|
375 | 362 | self.assertEqual(test_ns[k], v) |
|
376 | 363 | |
|
377 | 364 | def test_simple(self): |
|
378 | 365 | self.check_ns(['x=1'], dict(x=1)) |
|
379 | 366 | |
|
380 | 367 | def test_simple2(self): |
|
381 | 368 | self.check_ns(['if 1:', 'x=2'], dict(x=2)) |
|
382 | 369 | |
|
383 | 370 | def test_xy(self): |
|
384 | 371 | self.check_ns(['x=1; y=2'], dict(x=1, y=2)) |
|
385 | 372 | |
|
386 | 373 | def test_abc(self): |
|
387 | 374 | self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3)) |
|
388 | 375 | |
|
389 | 376 | def test_multi(self): |
|
390 | 377 | self.check_ns(['x =(1+','1+','2)'], dict(x=4)) |
|
391 | 378 | |
|
392 | 379 | |
|
393 | 380 | class IPythonInputTestCase(InputSplitterTestCase): |
|
394 | 381 | """By just creating a new class whose .isp is a different instance, we |
|
395 | 382 | re-run the same test battery on the new input splitter. |
|
396 | 383 | |
|
397 | 384 | In addition, this runs the tests over the syntax and syntax_ml dicts that |
|
398 | 385 | were tested by individual functions, as part of the OO interface. |
|
399 | 386 | |
|
400 | 387 | It also makes some checks on the raw buffer storage. |
|
401 | 388 | """ |
|
402 | 389 | |
|
403 | 390 | def setUp(self): |
|
404 | 391 | self.isp = isp.IPythonInputSplitter() |
|
405 | 392 | |
|
406 | 393 | def test_syntax(self): |
|
407 | 394 | """Call all single-line syntax tests from the main object""" |
|
408 | 395 | isp = self.isp |
|
409 | 396 | for example in syntax.values(): |
|
410 | 397 | for raw, out_t in example: |
|
411 | 398 | if raw.startswith(' '): |
|
412 | 399 | continue |
|
413 | 400 | |
|
414 | 401 | isp.push(raw+'\n') |
|
415 | 402 | out_raw = isp.source_raw |
|
416 | 403 | out = isp.source_reset() |
|
417 | 404 | self.assertEqual(out.rstrip(), out_t, |
|
418 | 405 | tt.pair_fail_msg.format("inputsplitter",raw, out_t, out)) |
|
419 | 406 | self.assertEqual(out_raw.rstrip(), raw.rstrip()) |
|
420 | 407 | |
|
421 | 408 | def test_syntax_multiline(self): |
|
422 | 409 | isp = self.isp |
|
423 | 410 | for example in syntax_ml.values(): |
|
424 | 411 | for line_pairs in example: |
|
425 | 412 | out_t_parts = [] |
|
426 | 413 | raw_parts = [] |
|
427 | 414 | for lraw, out_t_part in line_pairs: |
|
428 | 415 | if out_t_part is not None: |
|
429 | 416 | out_t_parts.append(out_t_part) |
|
430 | 417 | |
|
431 | 418 | if lraw is not None: |
|
432 | 419 | isp.push(lraw) |
|
433 | 420 | raw_parts.append(lraw) |
|
434 | 421 | |
|
435 | 422 | out_raw = isp.source_raw |
|
436 | 423 | out = isp.source_reset() |
|
437 | 424 | out_t = '\n'.join(out_t_parts).rstrip() |
|
438 | 425 | raw = '\n'.join(raw_parts).rstrip() |
|
439 | 426 | self.assertEqual(out.rstrip(), out_t) |
|
440 | 427 | self.assertEqual(out_raw.rstrip(), raw) |
|
441 | 428 | |
|
442 | 429 | def test_syntax_multiline_cell(self): |
|
443 | 430 | isp = self.isp |
|
444 | 431 | for example in syntax_ml.values(): |
|
445 | 432 | |
|
446 | 433 | out_t_parts = [] |
|
447 | 434 | for line_pairs in example: |
|
448 | 435 | raw = '\n'.join(r for r, _ in line_pairs if r is not None) |
|
449 | 436 | out_t = '\n'.join(t for _,t in line_pairs if t is not None) |
|
450 | 437 | out = isp.transform_cell(raw) |
|
451 | 438 | # Match ignoring trailing whitespace |
|
452 | 439 | self.assertEqual(out.rstrip(), out_t.rstrip()) |
|
453 | 440 | |
|
454 | 441 | def test_cellmagic_preempt(self): |
|
455 | 442 | isp = self.isp |
|
456 | 443 | for raw, name, line, cell in [ |
|
457 | 444 | ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'), |
|
458 | 445 | ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'), |
|
459 | 446 | (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'), |
|
460 | 447 | ("%%cellm \n>>> hi", u'cellm', u'', u'hi'), |
|
461 | 448 | ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'), |
|
462 | 449 | ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'), |
|
463 | 450 | ]: |
|
464 | 451 | expected = "get_ipython().run_cell_magic(%r, %r, %r)" % ( |
|
465 | 452 | name, line, cell |
|
466 | 453 | ) |
|
467 | 454 | out = isp.transform_cell(raw) |
|
468 | 455 | self.assertEqual(out.rstrip(), expected.rstrip()) |
|
456 | ||
|
457 | def test_multiline_passthrough(self): | |
|
458 | isp = self.isp | |
|
459 | class CommentTransformer(InputTransformer): | |
|
460 | def __init__(self): | |
|
461 | self._lines = [] | |
|
462 | ||
|
463 | def push(self, line): | |
|
464 | self._lines.append(line + '#') | |
|
465 | ||
|
466 | def reset(self): | |
|
467 | text = '\n'.join(self._lines) | |
|
468 | self._lines = [] | |
|
469 | return text | |
|
469 | 470 | |
|
471 | isp.physical_line_transforms.insert(0, CommentTransformer()) | |
|
470 | 472 | |
|
473 | for raw, expected in [ | |
|
474 | ("a=5", "a=5#"), | |
|
475 | ("%ls foo", "get_ipython().magic(%r)" % u'ls foo#'), | |
|
476 | ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().magic(%r)" % ( | |
|
477 | u'ls foo#', u'ls bar#' | |
|
478 | )), | |
|
479 | ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().magic(%r)\n4#\n5#" % u'ls foo#'), | |
|
480 | ]: | |
|
481 | out = isp.transform_cell(raw) | |
|
482 | self.assertEqual(out.rstrip(), expected.rstrip()) | |
|
471 | 483 | |
|
472 | 484 | #----------------------------------------------------------------------------- |
|
473 | 485 | # Main - use as a script, mostly for developer experiments |
|
474 | 486 | #----------------------------------------------------------------------------- |
|
475 | 487 | |
|
476 | 488 | if __name__ == '__main__': |
|
477 | 489 | # A simple demo for interactive experimentation. This code will not get |
|
478 | 490 | # picked up by any test suite. |
|
479 | 491 | from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter |
|
480 | 492 | |
|
481 | 493 | # configure here the syntax to use, prompt and whether to autoindent |
|
482 | 494 | #isp, start_prompt = InputSplitter(), '>>> ' |
|
483 | 495 | isp, start_prompt = IPythonInputSplitter(), 'In> ' |
|
484 | 496 | |
|
485 | 497 | autoindent = True |
|
486 | 498 | #autoindent = False |
|
487 | 499 | |
|
488 | 500 | try: |
|
489 | 501 | while True: |
|
490 | 502 | prompt = start_prompt |
|
491 | 503 | while isp.push_accepts_more(): |
|
492 | 504 | indent = ' '*isp.indent_spaces |
|
493 | 505 | if autoindent: |
|
494 | 506 | line = indent + input(prompt+indent) |
|
495 | 507 | else: |
|
496 | 508 | line = input(prompt) |
|
497 | 509 | isp.push(line) |
|
498 | 510 | prompt = '... ' |
|
499 | 511 | |
|
500 | 512 | # Here we just return input so we can use it in a test suite, but a |
|
501 | 513 | # real interpreter would instead send it for execution somewhere. |
|
502 | 514 | #src = isp.source; raise EOFError # dbg |
|
503 | 515 | raw = isp.source_raw |
|
504 | 516 | src = isp.source_reset() |
|
505 | 517 | print('Input source was:\n', src) |
|
506 | 518 | print('Raw source was:\n', raw) |
|
507 | 519 | except EOFError: |
|
508 | 520 | print('Bye') |
|
509 | 521 | |
|
510 | 522 | # Tests for cell magics support |
|
511 | 523 | |
|
512 | 524 | def test_last_blank(): |
|
513 | 525 | nt.assert_false(isp.last_blank('')) |
|
514 | 526 | nt.assert_false(isp.last_blank('abc')) |
|
515 | 527 | nt.assert_false(isp.last_blank('abc\n')) |
|
516 | 528 | nt.assert_false(isp.last_blank('abc\na')) |
|
517 | 529 | |
|
518 | 530 | nt.assert_true(isp.last_blank('\n')) |
|
519 | 531 | nt.assert_true(isp.last_blank('\n ')) |
|
520 | 532 | nt.assert_true(isp.last_blank('abc\n ')) |
|
521 | 533 | nt.assert_true(isp.last_blank('abc\n\n')) |
|
522 | 534 | nt.assert_true(isp.last_blank('abc\nd\n\n')) |
|
523 | 535 | nt.assert_true(isp.last_blank('abc\nd\ne\n\n')) |
|
524 | 536 | nt.assert_true(isp.last_blank('abc \n \n \n\n')) |
|
525 | 537 | |
|
526 | 538 | |
|
527 | 539 | def test_last_two_blanks(): |
|
528 | 540 | nt.assert_false(isp.last_two_blanks('')) |
|
529 | 541 | nt.assert_false(isp.last_two_blanks('abc')) |
|
530 | 542 | nt.assert_false(isp.last_two_blanks('abc\n')) |
|
531 | 543 | nt.assert_false(isp.last_two_blanks('abc\n\na')) |
|
532 | 544 | nt.assert_false(isp.last_two_blanks('abc\n \n')) |
|
533 | 545 | nt.assert_false(isp.last_two_blanks('abc\n\n')) |
|
534 | 546 | |
|
535 | 547 | nt.assert_true(isp.last_two_blanks('\n\n')) |
|
536 | 548 | nt.assert_true(isp.last_two_blanks('\n\n ')) |
|
537 | 549 | nt.assert_true(isp.last_two_blanks('\n \n')) |
|
538 | 550 | nt.assert_true(isp.last_two_blanks('abc\n\n ')) |
|
539 | 551 | nt.assert_true(isp.last_two_blanks('abc\n\n\n')) |
|
540 | 552 | nt.assert_true(isp.last_two_blanks('abc\n\n \n')) |
|
541 | 553 | nt.assert_true(isp.last_two_blanks('abc\n\n \n ')) |
|
542 | 554 | nt.assert_true(isp.last_two_blanks('abc\n\n \n \n')) |
|
543 | 555 | nt.assert_true(isp.last_two_blanks('abc\nd\n\n\n')) |
|
544 | 556 | nt.assert_true(isp.last_two_blanks('abc\nd\ne\nf\n\n\n')) |
|
545 | 557 | |
|
546 | 558 | |
|
547 | 559 | class CellMagicsCommon(object): |
|
548 | 560 | |
|
549 | 561 | def test_whole_cell(self): |
|
550 | 562 | src = "%%cellm line\nbody\n" |
|
551 | 563 | out = self.sp.transform_cell(src) |
|
552 | 564 | ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n" |
|
553 | 565 | nt.assert_equal(out, py3compat.u_format(ref)) |
|
554 | 566 | |
|
555 | 567 | def test_cellmagic_help(self): |
|
556 | 568 | self.sp.push('%%cellm?') |
|
557 | 569 | nt.assert_false(self.sp.push_accepts_more()) |
|
558 | 570 | |
|
559 | 571 | def tearDown(self): |
|
560 | 572 | self.sp.reset() |
|
561 | 573 | |
|
562 | 574 | |
|
563 | 575 | class CellModeCellMagics(CellMagicsCommon, unittest.TestCase): |
|
564 | 576 | sp = isp.IPythonInputSplitter(line_input_checker=False) |
|
565 | 577 | |
|
566 | 578 | def test_incremental(self): |
|
567 | 579 | sp = self.sp |
|
568 | 580 | sp.push('%%cellm firstline\n') |
|
569 | 581 | nt.assert_true(sp.push_accepts_more()) #1 |
|
570 | 582 | sp.push('line2\n') |
|
571 | 583 | nt.assert_true(sp.push_accepts_more()) #2 |
|
572 | 584 | sp.push('\n') |
|
573 | 585 | # This should accept a blank line and carry on until the cell is reset |
|
574 | 586 | nt.assert_true(sp.push_accepts_more()) #3 |
|
575 | 587 | |
|
576 | 588 | class LineModeCellMagics(CellMagicsCommon, unittest.TestCase): |
|
577 | 589 | sp = isp.IPythonInputSplitter(line_input_checker=True) |
|
578 | 590 | |
|
579 | 591 | def test_incremental(self): |
|
580 | 592 | sp = self.sp |
|
581 | 593 | sp.push('%%cellm line2\n') |
|
582 | 594 | nt.assert_true(sp.push_accepts_more()) #1 |
|
583 | 595 | sp.push('\n') |
|
584 | 596 | # In this case, a blank line should end the cell magic |
|
585 | 597 | nt.assert_false(sp.push_accepts_more()) #2 |
General Comments 0
You need to be logged in to leave comments.
Login now