##// END OF EJS Templates
always pass single lines to transform.push...
MinRK -
Show More
@@ -1,638 +1,646 b''
1 1 """Input handling and transformation machinery.
2 2
3 3 The first class in this module, :class:`InputSplitter`, is designed to tell when
4 4 input from a line-oriented frontend is complete and should be executed, and when
5 5 the user should be prompted for another line of code instead. The name 'input
6 6 splitter' is largely for historical reasons.
7 7
8 8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 9 with full support for the extended IPython syntax (magics, system calls, etc).
10 10 The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
11 11 :class:`IPythonInputSplitter` feeds the raw code to the transformers in order
12 12 and stores the results.
13 13
14 14 For more details, see the class docstrings below.
15
16 Authors
17 -------
18
19 * Fernando Perez
20 * Brian Granger
21 * Thomas Kluyver
22 15 """
23 #-----------------------------------------------------------------------------
24 # Copyright (C) 2010 The IPython Development Team
25 #
26 # Distributed under the terms of the BSD License. The full license is in
27 # the file COPYING, distributed as part of this software.
28 #-----------------------------------------------------------------------------
29 16
30 #-----------------------------------------------------------------------------
31 # Imports
32 #-----------------------------------------------------------------------------
33 # stdlib
17 # Copyright (c) IPython Development Team.
18 # Distributed under the terms of the Modified BSD License.
34 19 import ast
35 20 import codeop
36 21 import re
37 22 import sys
38 23
39 # IPython modules
40 24 from IPython.utils.py3compat import cast_unicode
41 25 from IPython.core.inputtransformer import (leading_indent,
42 26 classic_prompt,
43 27 ipy_prompt,
44 28 strip_encoding_cookie,
45 29 cellmagic,
46 30 assemble_logical_lines,
47 31 help_end,
48 32 escaped_commands,
49 33 assign_from_magic,
50 34 assign_from_system,
51 35 assemble_python_lines,
52 36 )
53 37
54 38 # These are available in this module for backwards compatibility.
55 39 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
56 40 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
57 41 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
58 42
59 43 #-----------------------------------------------------------------------------
60 44 # Utilities
61 45 #-----------------------------------------------------------------------------
62 46
63 47 # FIXME: These are general-purpose utilities that later can be moved to the
64 48 # general ward. Kept here for now because we're being very strict about test
65 49 # coverage with this code, and this lets us ensure that we keep 100% coverage
66 50 # while developing.
67 51
68 52 # compiled regexps for autoindent management
69 53 dedent_re = re.compile('|'.join([
70 54 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
71 55 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
72 56 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
73 57 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
74 58 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
75 59 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
76 60 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
77 61 ]))
78 62 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
79 63
80 64 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
81 65 # before pure comments
82 66 comment_line_re = re.compile('^\s*\#')
83 67
84 68
85 69 def num_ini_spaces(s):
86 70 """Return the number of initial spaces in a string.
87 71
88 72 Note that tabs are counted as a single space. For now, we do *not* support
89 73 mixing of tabs and spaces in the user's input.
90 74
91 75 Parameters
92 76 ----------
93 77 s : string
94 78
95 79 Returns
96 80 -------
97 81 n : int
98 82 """
99 83
100 84 ini_spaces = ini_spaces_re.match(s)
101 85 if ini_spaces:
102 86 return ini_spaces.end()
103 87 else:
104 88 return 0
105 89
106 90 def last_blank(src):
107 91 """Determine if the input source ends in a blank.
108 92
109 93 A blank is either a newline or a line consisting of whitespace.
110 94
111 95 Parameters
112 96 ----------
113 97 src : string
114 98 A single or multiline string.
115 99 """
116 100 if not src: return False
117 101 ll = src.splitlines()[-1]
118 102 return (ll == '') or ll.isspace()
119 103
120 104
121 105 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
122 106 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
123 107
124 108 def last_two_blanks(src):
125 109 """Determine if the input source ends in two blanks.
126 110
127 111 A blank is either a newline or a line consisting of whitespace.
128 112
129 113 Parameters
130 114 ----------
131 115 src : string
132 116 A single or multiline string.
133 117 """
134 118 if not src: return False
135 119 # The logic here is tricky: I couldn't get a regexp to work and pass all
136 120 # the tests, so I took a different approach: split the source by lines,
137 121 # grab the last two and prepend '###\n' as a stand-in for whatever was in
138 122 # the body before the last two lines. Then, with that structure, it's
139 123 # possible to analyze with two regexps. Not the most elegant solution, but
140 124 # it works. If anyone tries to change this logic, make sure to validate
141 125 # the whole test suite first!
142 126 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
143 127 return (bool(last_two_blanks_re.match(new_src)) or
144 128 bool(last_two_blanks_re2.match(new_src)) )
145 129
146 130
147 131 def remove_comments(src):
148 132 """Remove all comments from input source.
149 133
150 134 Note: comments are NOT recognized inside of strings!
151 135
152 136 Parameters
153 137 ----------
154 138 src : string
155 139 A single or multiline input string.
156 140
157 141 Returns
158 142 -------
159 143 String with all Python comments removed.
160 144 """
161 145
162 146 return re.sub('#.*', '', src)
163 147
164 148
165 149 def get_input_encoding():
166 150 """Return the default standard input encoding.
167 151
168 152 If sys.stdin has no encoding, 'ascii' is returned."""
169 153 # There are strange environments for which sys.stdin.encoding is None. We
170 154 # ensure that a valid encoding is returned.
171 155 encoding = getattr(sys.stdin, 'encoding', None)
172 156 if encoding is None:
173 157 encoding = 'ascii'
174 158 return encoding
175 159
176 160 #-----------------------------------------------------------------------------
177 161 # Classes and functions for normal Python syntax handling
178 162 #-----------------------------------------------------------------------------
179 163
180 164 class InputSplitter(object):
181 165 r"""An object that can accumulate lines of Python source before execution.
182 166
183 167 This object is designed to be fed python source line-by-line, using
184 168 :meth:`push`. It will return on each push whether the currently pushed
185 169 code could be executed already. In addition, it provides a method called
186 170 :meth:`push_accepts_more` that can be used to query whether more input
187 171 can be pushed into a single interactive block.
188 172
189 173 This is a simple example of how an interactive terminal-based client can use
190 174 this tool::
191 175
192 176 isp = InputSplitter()
193 177 while isp.push_accepts_more():
194 178 indent = ' '*isp.indent_spaces
195 179 prompt = '>>> ' + indent
196 180 line = indent + raw_input(prompt)
197 181 isp.push(line)
198 182 print 'Input source was:\n', isp.source_reset(),
199 183 """
200 184 # Number of spaces of indentation computed from input that has been pushed
201 185 # so far. This is the attributes callers should query to get the current
202 186 # indentation level, in order to provide auto-indent facilities.
203 187 indent_spaces = 0
204 188 # String, indicating the default input encoding. It is computed by default
205 189 # at initialization time via get_input_encoding(), but it can be reset by a
206 190 # client with specific knowledge of the encoding.
207 191 encoding = ''
208 192 # String where the current full source input is stored, properly encoded.
209 193 # Reading this attribute is the normal way of querying the currently pushed
210 194 # source code, that has been properly encoded.
211 195 source = ''
212 196 # Code object corresponding to the current source. It is automatically
213 197 # synced to the source, so it can be queried at any time to obtain the code
214 198 # object; it will be None if the source doesn't compile to valid Python.
215 199 code = None
216 200
217 201 # Private attributes
218 202
219 203 # List with lines of input accumulated so far
220 204 _buffer = None
221 205 # Command compiler
222 206 _compile = None
223 207 # Mark when input has changed indentation all the way back to flush-left
224 208 _full_dedent = False
225 209 # Boolean indicating whether the current block is complete
226 210 _is_complete = None
227 211
228 212 def __init__(self):
229 213 """Create a new InputSplitter instance.
230 214 """
231 215 self._buffer = []
232 216 self._compile = codeop.CommandCompiler()
233 217 self.encoding = get_input_encoding()
234 218
235 219 def reset(self):
236 220 """Reset the input buffer and associated state."""
237 221 self.indent_spaces = 0
238 222 self._buffer[:] = []
239 223 self.source = ''
240 224 self.code = None
241 225 self._is_complete = False
242 226 self._full_dedent = False
243 227
244 228 def source_reset(self):
245 229 """Return the input source and perform a full reset.
246 230 """
247 231 out = self.source
248 232 self.reset()
249 233 return out
250 234
251 235 def push(self, lines):
252 236 """Push one or more lines of input.
253 237
254 238 This stores the given lines and returns a status code indicating
255 239 whether the code forms a complete Python block or not.
256 240
257 241 Any exceptions generated in compilation are swallowed, but if an
258 242 exception was produced, the method returns True.
259 243
260 244 Parameters
261 245 ----------
262 246 lines : string
263 247 One or more lines of Python input.
264 248
265 249 Returns
266 250 -------
267 251 is_complete : boolean
268 252 True if the current input source (the result of the current input
269 253 plus prior inputs) forms a complete Python execution block. Note that
270 254 this value is also stored as a private attribute (``_is_complete``), so it
271 255 can be queried at any time.
272 256 """
273 257 self._store(lines)
274 258 source = self.source
275 259
276 260 # Before calling _compile(), reset the code object to None so that if an
277 261 # exception is raised in compilation, we don't mislead by having
278 262 # inconsistent code/source attributes.
279 263 self.code, self._is_complete = None, None
280 264
281 265 # Honor termination lines properly
282 266 if source.endswith('\\\n'):
283 267 return False
284 268
285 269 self._update_indent(lines)
286 270 try:
287 271 self.code = self._compile(source, symbol="exec")
288 272 # Invalid syntax can produce any of a number of different errors from
289 273 # inside the compiler, so we have to catch them all. Syntax errors
290 274 # immediately produce a 'ready' block, so the invalid Python can be
291 275 # sent to the kernel for evaluation with possible ipython
292 276 # special-syntax conversion.
293 277 except (SyntaxError, OverflowError, ValueError, TypeError,
294 278 MemoryError):
295 279 self._is_complete = True
296 280 else:
297 281 # Compilation didn't produce any exceptions (though it may not have
298 282 # given a complete code object)
299 283 self._is_complete = self.code is not None
300 284
301 285 return self._is_complete
302 286
303 287 def push_accepts_more(self):
304 288 """Return whether a block of interactive input can accept more input.
305 289
306 290 This method is meant to be used by line-oriented frontends, who need to
307 291 guess whether a block is complete or not based solely on prior and
308 292 current input lines. The InputSplitter considers it has a complete
309 293 interactive block and will not accept more input when either:
310 294
311 295 * A SyntaxError is raised
312 296
313 297 * The code is complete and consists of a single line or a single
314 298 non-compound statement
315 299
316 300 * The code is complete and has a blank line at the end
317 301
318 302 If the current input produces a syntax error, this method immediately
319 303 returns False but does *not* raise the syntax error exception, as
320 304 typically clients will want to send invalid syntax to an execution
321 305 backend which might convert the invalid syntax into valid Python via
322 306 one of the dynamic IPython mechanisms.
323 307 """
324 308
325 309 # With incomplete input, unconditionally accept more
326 310 # A syntax error also sets _is_complete to True - see push()
327 311 if not self._is_complete:
328 312 #print("Not complete") # debug
329 313 return True
330 314
331 315 # The user can make any (complete) input execute by leaving a blank line
332 316 last_line = self.source.splitlines()[-1]
333 317 if (not last_line) or last_line.isspace():
334 318 #print("Blank line") # debug
335 319 return False
336 320
337 321 # If there's just a single line or AST node, and we're flush left, as is
338 322 # the case after a simple statement such as 'a=1', we want to execute it
339 323 # straight away.
340 324 if self.indent_spaces==0:
341 325 if len(self.source.splitlines()) <= 1:
342 326 return False
343 327
344 328 try:
345 329 code_ast = ast.parse(u''.join(self._buffer))
346 330 except Exception:
347 331 #print("Can't parse AST") # debug
348 332 return False
349 333 else:
350 334 if len(code_ast.body) == 1 and \
351 335 not hasattr(code_ast.body[0], 'body'):
352 336 #print("Simple statement") # debug
353 337 return False
354 338
355 339 # General fallback - accept more code
356 340 return True
357 341
358 342 #------------------------------------------------------------------------
359 343 # Private interface
360 344 #------------------------------------------------------------------------
361 345
362 346 def _find_indent(self, line):
363 347 """Compute the new indentation level for a single line.
364 348
365 349 Parameters
366 350 ----------
367 351 line : str
368 352 A single new line of non-whitespace, non-comment Python input.
369 353
370 354 Returns
371 355 -------
372 356 indent_spaces : int
373 357 New value for the indent level (it may be equal to self.indent_spaces
374 358 if indentation doesn't change.
375 359
376 360 full_dedent : boolean
377 361 Whether the new line causes a full flush-left dedent.
378 362 """
379 363 indent_spaces = self.indent_spaces
380 364 full_dedent = self._full_dedent
381 365
382 366 inisp = num_ini_spaces(line)
383 367 if inisp < indent_spaces:
384 368 indent_spaces = inisp
385 369 if indent_spaces <= 0:
386 370 #print 'Full dedent in text',self.source # dbg
387 371 full_dedent = True
388 372
389 373 if line.rstrip()[-1] == ':':
390 374 indent_spaces += 4
391 375 elif dedent_re.match(line):
392 376 indent_spaces -= 4
393 377 if indent_spaces <= 0:
394 378 full_dedent = True
395 379
396 380 # Safety
397 381 if indent_spaces < 0:
398 382 indent_spaces = 0
399 383 #print 'safety' # dbg
400 384
401 385 return indent_spaces, full_dedent
402 386
403 387 def _update_indent(self, lines):
404 388 for line in remove_comments(lines).splitlines():
405 389 if line and not line.isspace():
406 390 self.indent_spaces, self._full_dedent = self._find_indent(line)
407 391
408 392 def _store(self, lines, buffer=None, store='source'):
409 393 """Store one or more lines of input.
410 394
411 395 If input lines are not newline-terminated, a newline is automatically
412 396 appended."""
413 397
414 398 if buffer is None:
415 399 buffer = self._buffer
416 400
417 401 if lines.endswith('\n'):
418 402 buffer.append(lines)
419 403 else:
420 404 buffer.append(lines+'\n')
421 405 setattr(self, store, self._set_source(buffer))
422 406
423 407 def _set_source(self, buffer):
424 408 return u''.join(buffer)
425 409
426 410
427 411 class IPythonInputSplitter(InputSplitter):
428 412 """An input splitter that recognizes all of IPython's special syntax."""
429 413
430 414 # String with raw, untransformed input.
431 415 source_raw = ''
432 416
433 417 # Flag to track when a transformer has stored input that it hasn't given
434 418 # back yet.
435 419 transformer_accumulating = False
436 420
437 421 # Flag to track when assemble_python_lines has stored input that it hasn't
438 422 # given back yet.
439 423 within_python_line = False
440 424
441 425 # Private attributes
442 426
443 427 # List with lines of raw input accumulated so far.
444 428 _buffer_raw = None
445 429
446 430 def __init__(self, line_input_checker=True, physical_line_transforms=None,
447 431 logical_line_transforms=None, python_line_transforms=None):
448 432 super(IPythonInputSplitter, self).__init__()
449 433 self._buffer_raw = []
450 434 self._validate = True
451 435
452 436 if physical_line_transforms is not None:
453 437 self.physical_line_transforms = physical_line_transforms
454 438 else:
455 439 self.physical_line_transforms = [
456 440 leading_indent(),
457 441 classic_prompt(),
458 442 ipy_prompt(),
459 443 strip_encoding_cookie(),
460 444 cellmagic(end_on_blank_line=line_input_checker),
461 445 ]
462 446
463 447 self.assemble_logical_lines = assemble_logical_lines()
464 448 if logical_line_transforms is not None:
465 449 self.logical_line_transforms = logical_line_transforms
466 450 else:
467 451 self.logical_line_transforms = [
468 452 help_end(),
469 453 escaped_commands(),
470 454 assign_from_magic(),
471 455 assign_from_system(),
472 456 ]
473 457
474 458 self.assemble_python_lines = assemble_python_lines()
475 459 if python_line_transforms is not None:
476 460 self.python_line_transforms = python_line_transforms
477 461 else:
478 462 # We don't use any of these at present
479 463 self.python_line_transforms = []
480 464
481 465 @property
482 466 def transforms(self):
483 467 "Quick access to all transformers."
484 468 return self.physical_line_transforms + \
485 469 [self.assemble_logical_lines] + self.logical_line_transforms + \
486 470 [self.assemble_python_lines] + self.python_line_transforms
487 471
488 472 @property
489 473 def transforms_in_use(self):
490 474 """Transformers, excluding logical line transformers if we're in a
491 475 Python line."""
492 476 t = self.physical_line_transforms[:]
493 477 if not self.within_python_line:
494 478 t += [self.assemble_logical_lines] + self.logical_line_transforms
495 479 return t + [self.assemble_python_lines] + self.python_line_transforms
496 480
497 481 def reset(self):
498 482 """Reset the input buffer and associated state."""
499 483 super(IPythonInputSplitter, self).reset()
500 484 self._buffer_raw[:] = []
501 485 self.source_raw = ''
502 486 self.transformer_accumulating = False
503 487 self.within_python_line = False
504 488
505 489 for t in self.transforms:
506 490 try:
507 491 t.reset()
508 492 except SyntaxError:
509 493 # Nothing that calls reset() expects to handle transformer
510 494 # errors
511 495 pass
512 496
513 497 def flush_transformers(self):
514 498 def _flush(transform, out):
515 if out is not None:
516 tmp = transform.push(out)
517 return tmp or transform.reset() or None
518 else:
519 return transform.reset() or None
499 """yield transformed lines
500
501 always strings, never None
502
503 transform: the current transform
504 out: an iterable of previously transformed inputs.
505 Each may be multiline, which will be passed
506 one line at a time to transform.
507 """
508 anything = False
509 for out in out:
510 anything = True
511 tmp = None
512 for line in out.splitlines():
513 # push one line at a time
514 tmp = transform.push(line)
515 if tmp is not None:
516 yield tmp
517 if tmp is None:
518 # transformer is still consuming, reset
519 tmp = transform.reset()
520 if tmp is not None:
521 yield tmp
522 if not anything:
523 # nothing was pushed, reset
524 tmp = transform.reset()
525 if tmp is not None:
526 yield tmp
520 527
521 out = None
528 out = []
522 529 for t in self.transforms_in_use:
523 530 out = _flush(t, out)
524 531
525 if out is not None:
526 self._store(out)
532 out = list(out)
533 if out:
534 self._store('\n'.join(out))
527 535
528 536 def raw_reset(self):
529 537 """Return raw input only and perform a full reset.
530 538 """
531 539 out = self.source_raw
532 540 self.reset()
533 541 return out
534 542
535 543 def source_reset(self):
536 544 try:
537 545 self.flush_transformers()
538 546 return self.source
539 547 finally:
540 548 self.reset()
541 549
542 550 def push_accepts_more(self):
543 551 if self.transformer_accumulating:
544 552 return True
545 553 else:
546 554 return super(IPythonInputSplitter, self).push_accepts_more()
547 555
548 556 def transform_cell(self, cell):
549 557 """Process and translate a cell of input.
550 558 """
551 559 self.reset()
552 560 try:
553 561 self.push(cell)
554 562 self.flush_transformers()
555 563 return self.source
556 564 finally:
557 565 self.reset()
558 566
559 567 def push(self, lines):
560 568 """Push one or more lines of IPython input.
561 569
562 570 This stores the given lines and returns a status code indicating
563 571 whether the code forms a complete Python block or not, after processing
564 572 all input lines for special IPython syntax.
565 573
566 574 Any exceptions generated in compilation are swallowed, but if an
567 575 exception was produced, the method returns True.
568 576
569 577 Parameters
570 578 ----------
571 579 lines : string
572 580 One or more lines of Python input.
573 581
574 582 Returns
575 583 -------
576 584 is_complete : boolean
577 585 True if the current input source (the result of the current input
578 586 plus prior inputs) forms a complete Python execution block. Note that
579 587 this value is also stored as a private attribute (_is_complete), so it
580 588 can be queried at any time.
581 589 """
582 590
583 591 # We must ensure all input is pure unicode
584 592 lines = cast_unicode(lines, self.encoding)
585 593
586 594 # ''.splitlines() --> [], but we need to push the empty line to transformers
587 595 lines_list = lines.splitlines()
588 596 if not lines_list:
589 597 lines_list = ['']
590 598
591 599 # Store raw source before applying any transformations to it. Note
592 600 # that this must be done *after* the reset() call that would otherwise
593 601 # flush the buffer.
594 602 self._store(lines, self._buffer_raw, 'source_raw')
595 603
596 604 for line in lines_list:
597 605 out = self.push_line(line)
598 606
599 607 return out
600 608
601 609 def push_line(self, line):
602 610 buf = self._buffer
603 611
604 612 def _accumulating(dbg):
605 613 #print(dbg)
606 614 self.transformer_accumulating = True
607 615 return False
608 616
609 617 for transformer in self.physical_line_transforms:
610 618 line = transformer.push(line)
611 619 if line is None:
612 620 return _accumulating(transformer)
613 621
614 622 if not self.within_python_line:
615 623 line = self.assemble_logical_lines.push(line)
616 624 if line is None:
617 625 return _accumulating('acc logical line')
618 626
619 627 for transformer in self.logical_line_transforms:
620 628 line = transformer.push(line)
621 629 if line is None:
622 630 return _accumulating(transformer)
623 631
624 632 line = self.assemble_python_lines.push(line)
625 633 if line is None:
626 634 self.within_python_line = True
627 635 return _accumulating('acc python line')
628 636 else:
629 637 self.within_python_line = False
630 638
631 639 for transformer in self.python_line_transforms:
632 640 line = transformer.push(line)
633 641 if line is None:
634 642 return _accumulating(transformer)
635 643
636 644 #print("transformers clear") #debug
637 645 self.transformer_accumulating = False
638 646 return super(IPythonInputSplitter, self).push(line)
@@ -1,585 +1,597 b''
1 1 # -*- coding: utf-8 -*-
2 """Tests for the inputsplitter module.
2 """Tests for the inputsplitter module."""
3 3
4 Authors
5 -------
6 * Fernando Perez
7 * Robert Kern
8 """
9 4 from __future__ import print_function
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2010-2011 The IPython Development Team
12 #
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
15 #-----------------------------------------------------------------------------
16 5
17 #-----------------------------------------------------------------------------
18 # Imports
19 #-----------------------------------------------------------------------------
20 # stdlib
6 # Copyright (c) IPython Development Team.
7 # Distributed under the terms of the Modified BSD License.
8
21 9 import unittest
22 10 import sys
23 11
24 # Third party
25 12 import nose.tools as nt
26 13
27 # Our own
28 14 from IPython.core import inputsplitter as isp
15 from IPython.core.inputtransformer import InputTransformer
29 16 from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
30 17 from IPython.testing import tools as tt
31 18 from IPython.utils import py3compat
32 19 from IPython.utils.py3compat import string_types, input
33 20
34 21 #-----------------------------------------------------------------------------
35 22 # Semi-complete examples (also used as tests)
36 23 #-----------------------------------------------------------------------------
37 24
38 25 # Note: at the bottom, there's a slightly more complete version of this that
39 26 # can be useful during development of code here.
40 27
41 28 def mini_interactive_loop(input_func):
42 29 """Minimal example of the logic of an interactive interpreter loop.
43 30
44 31 This serves as an example, and it is used by the test system with a fake
45 32 raw_input that simulates interactive input."""
46 33
47 34 from IPython.core.inputsplitter import InputSplitter
48 35
49 36 isp = InputSplitter()
50 37 # In practice, this input loop would be wrapped in an outside loop to read
51 38 # input indefinitely, until some exit/quit command was issued. Here we
52 39 # only illustrate the basic inner loop.
53 40 while isp.push_accepts_more():
54 41 indent = ' '*isp.indent_spaces
55 42 prompt = '>>> ' + indent
56 43 line = indent + input_func(prompt)
57 44 isp.push(line)
58 45
59 46 # Here we just return input so we can use it in a test suite, but a real
60 47 # interpreter would instead send it for execution somewhere.
61 48 src = isp.source_reset()
62 49 #print 'Input source was:\n', src # dbg
63 50 return src
64 51
65 52 #-----------------------------------------------------------------------------
66 53 # Test utilities, just for local use
67 54 #-----------------------------------------------------------------------------
68 55
69 56 def assemble(block):
70 57 """Assemble a block into multi-line sub-blocks."""
71 58 return ['\n'.join(sub_block)+'\n' for sub_block in block]
72 59
73 60
74 61 def pseudo_input(lines):
75 62 """Return a function that acts like raw_input but feeds the input list."""
76 63 ilines = iter(lines)
77 64 def raw_in(prompt):
78 65 try:
79 66 return next(ilines)
80 67 except StopIteration:
81 68 return ''
82 69 return raw_in
83 70
84 71 #-----------------------------------------------------------------------------
85 72 # Tests
86 73 #-----------------------------------------------------------------------------
87 74 def test_spaces():
88 75 tests = [('', 0),
89 76 (' ', 1),
90 77 ('\n', 0),
91 78 (' \n', 1),
92 79 ('x', 0),
93 80 (' x', 1),
94 81 (' x',2),
95 82 (' x',4),
96 83 # Note: tabs are counted as a single whitespace!
97 84 ('\tx', 1),
98 85 ('\t x', 2),
99 86 ]
100 87 tt.check_pairs(isp.num_ini_spaces, tests)
101 88
102 89
103 90 def test_remove_comments():
104 91 tests = [('text', 'text'),
105 92 ('text # comment', 'text '),
106 93 ('text # comment\n', 'text \n'),
107 94 ('text # comment \n', 'text \n'),
108 95 ('line # c \nline\n','line \nline\n'),
109 96 ('line # c \nline#c2 \nline\nline #c\n\n',
110 97 'line \nline\nline\nline \n\n'),
111 98 ]
112 99 tt.check_pairs(isp.remove_comments, tests)
113 100
114 101
115 102 def test_get_input_encoding():
116 103 encoding = isp.get_input_encoding()
117 104 nt.assert_true(isinstance(encoding, string_types))
118 105 # simple-minded check that at least encoding a simple string works with the
119 106 # encoding we got.
120 107 nt.assert_equal(u'test'.encode(encoding), b'test')
121 108
122 109
123 110 class NoInputEncodingTestCase(unittest.TestCase):
124 111 def setUp(self):
125 112 self.old_stdin = sys.stdin
126 113 class X: pass
127 114 fake_stdin = X()
128 115 sys.stdin = fake_stdin
129 116
130 117 def test(self):
131 118 # Verify that if sys.stdin has no 'encoding' attribute we do the right
132 119 # thing
133 120 enc = isp.get_input_encoding()
134 121 self.assertEqual(enc, 'ascii')
135 122
136 123 def tearDown(self):
137 124 sys.stdin = self.old_stdin
138 125
139 126
140 127 class InputSplitterTestCase(unittest.TestCase):
141 128 def setUp(self):
142 129 self.isp = isp.InputSplitter()
143 130
144 131 def test_reset(self):
145 132 isp = self.isp
146 133 isp.push('x=1')
147 134 isp.reset()
148 135 self.assertEqual(isp._buffer, [])
149 136 self.assertEqual(isp.indent_spaces, 0)
150 137 self.assertEqual(isp.source, '')
151 138 self.assertEqual(isp.code, None)
152 139 self.assertEqual(isp._is_complete, False)
153 140
154 141 def test_source(self):
155 142 self.isp._store('1')
156 143 self.isp._store('2')
157 144 self.assertEqual(self.isp.source, '1\n2\n')
158 145 self.assertTrue(len(self.isp._buffer)>0)
159 146 self.assertEqual(self.isp.source_reset(), '1\n2\n')
160 147 self.assertEqual(self.isp._buffer, [])
161 148 self.assertEqual(self.isp.source, '')
162 149
163 150 def test_indent(self):
164 151 isp = self.isp # shorthand
165 152 isp.push('x=1')
166 153 self.assertEqual(isp.indent_spaces, 0)
167 154 isp.push('if 1:\n x=1')
168 155 self.assertEqual(isp.indent_spaces, 4)
169 156 isp.push('y=2\n')
170 157 self.assertEqual(isp.indent_spaces, 0)
171 158
172 159 def test_indent2(self):
173 160 isp = self.isp
174 161 isp.push('if 1:')
175 162 self.assertEqual(isp.indent_spaces, 4)
176 163 isp.push(' x=1')
177 164 self.assertEqual(isp.indent_spaces, 4)
178 165 # Blank lines shouldn't change the indent level
179 166 isp.push(' '*2)
180 167 self.assertEqual(isp.indent_spaces, 4)
181 168
182 169 def test_indent3(self):
183 170 isp = self.isp
184 171 # When a multiline statement contains parens or multiline strings, we
185 172 # shouldn't get confused.
186 173 isp.push("if 1:")
187 174 isp.push(" x = (1+\n 2)")
188 175 self.assertEqual(isp.indent_spaces, 4)
189 176
190 177 def test_indent4(self):
191 178 isp = self.isp
192 179 # whitespace after ':' should not screw up indent level
193 180 isp.push('if 1: \n x=1')
194 181 self.assertEqual(isp.indent_spaces, 4)
195 182 isp.push('y=2\n')
196 183 self.assertEqual(isp.indent_spaces, 0)
197 184 isp.push('if 1:\t\n x=1')
198 185 self.assertEqual(isp.indent_spaces, 4)
199 186 isp.push('y=2\n')
200 187 self.assertEqual(isp.indent_spaces, 0)
201 188
202 189 def test_dedent_pass(self):
203 190 isp = self.isp # shorthand
204 191 # should NOT cause dedent
205 192 isp.push('if 1:\n passes = 5')
206 193 self.assertEqual(isp.indent_spaces, 4)
207 194 isp.push('if 1:\n pass')
208 195 self.assertEqual(isp.indent_spaces, 0)
209 196 isp.push('if 1:\n pass ')
210 197 self.assertEqual(isp.indent_spaces, 0)
211 198
212 199 def test_dedent_break(self):
213 200 isp = self.isp # shorthand
214 201 # should NOT cause dedent
215 202 isp.push('while 1:\n breaks = 5')
216 203 self.assertEqual(isp.indent_spaces, 4)
217 204 isp.push('while 1:\n break')
218 205 self.assertEqual(isp.indent_spaces, 0)
219 206 isp.push('while 1:\n break ')
220 207 self.assertEqual(isp.indent_spaces, 0)
221 208
222 209 def test_dedent_continue(self):
223 210 isp = self.isp # shorthand
224 211 # should NOT cause dedent
225 212 isp.push('while 1:\n continues = 5')
226 213 self.assertEqual(isp.indent_spaces, 4)
227 214 isp.push('while 1:\n continue')
228 215 self.assertEqual(isp.indent_spaces, 0)
229 216 isp.push('while 1:\n continue ')
230 217 self.assertEqual(isp.indent_spaces, 0)
231 218
232 219 def test_dedent_raise(self):
233 220 isp = self.isp # shorthand
234 221 # should NOT cause dedent
235 222 isp.push('if 1:\n raised = 4')
236 223 self.assertEqual(isp.indent_spaces, 4)
237 224 isp.push('if 1:\n raise TypeError()')
238 225 self.assertEqual(isp.indent_spaces, 0)
239 226 isp.push('if 1:\n raise')
240 227 self.assertEqual(isp.indent_spaces, 0)
241 228 isp.push('if 1:\n raise ')
242 229 self.assertEqual(isp.indent_spaces, 0)
243 230
244 231 def test_dedent_return(self):
245 232 isp = self.isp # shorthand
246 233 # should NOT cause dedent
247 234 isp.push('if 1:\n returning = 4')
248 235 self.assertEqual(isp.indent_spaces, 4)
249 236 isp.push('if 1:\n return 5 + 493')
250 237 self.assertEqual(isp.indent_spaces, 0)
251 238 isp.push('if 1:\n return')
252 239 self.assertEqual(isp.indent_spaces, 0)
253 240 isp.push('if 1:\n return ')
254 241 self.assertEqual(isp.indent_spaces, 0)
255 242 isp.push('if 1:\n return(0)')
256 243 self.assertEqual(isp.indent_spaces, 0)
257 244
258 245 def test_push(self):
259 246 isp = self.isp
260 247 self.assertTrue(isp.push('x=1'))
261 248
262 249 def test_push2(self):
263 250 isp = self.isp
264 251 self.assertFalse(isp.push('if 1:'))
265 252 for line in [' x=1', '# a comment', ' y=2']:
266 253 print(line)
267 254 self.assertTrue(isp.push(line))
268 255
269 256 def test_push3(self):
270 257 isp = self.isp
271 258 isp.push('if True:')
272 259 isp.push(' a = 1')
273 260 self.assertFalse(isp.push('b = [1,'))
274 261
275 262 def test_push_accepts_more(self):
276 263 isp = self.isp
277 264 isp.push('x=1')
278 265 self.assertFalse(isp.push_accepts_more())
279 266
280 267 def test_push_accepts_more2(self):
281 268 isp = self.isp
282 269 isp.push('if 1:')
283 270 self.assertTrue(isp.push_accepts_more())
284 271 isp.push(' x=1')
285 272 self.assertTrue(isp.push_accepts_more())
286 273 isp.push('')
287 274 self.assertFalse(isp.push_accepts_more())
288 275
289 276 def test_push_accepts_more3(self):
290 277 isp = self.isp
291 278 isp.push("x = (2+\n3)")
292 279 self.assertFalse(isp.push_accepts_more())
293 280
294 281 def test_push_accepts_more4(self):
295 282 isp = self.isp
296 283 # When a multiline statement contains parens or multiline strings, we
297 284 # shouldn't get confused.
298 285 # FIXME: we should be able to better handle de-dents in statements like
299 286 # multiline strings and multiline expressions (continued with \ or
300 287 # parens). Right now we aren't handling the indentation tracking quite
301 288 # correctly with this, though in practice it may not be too much of a
302 289 # problem. We'll need to see.
303 290 isp.push("if 1:")
304 291 isp.push(" x = (2+")
305 292 isp.push(" 3)")
306 293 self.assertTrue(isp.push_accepts_more())
307 294 isp.push(" y = 3")
308 295 self.assertTrue(isp.push_accepts_more())
309 296 isp.push('')
310 297 self.assertFalse(isp.push_accepts_more())
311 298
312 299 def test_push_accepts_more5(self):
313 300 isp = self.isp
314 301 isp.push('try:')
315 302 isp.push(' a = 5')
316 303 isp.push('except:')
317 304 isp.push(' raise')
318 305 # We want to be able to add an else: block at this point, so it should
319 306 # wait for a blank line.
320 307 self.assertTrue(isp.push_accepts_more())
321 308
322 309 def test_continuation(self):
323 310 isp = self.isp
324 311 isp.push("import os, \\")
325 312 self.assertTrue(isp.push_accepts_more())
326 313 isp.push("sys")
327 314 self.assertFalse(isp.push_accepts_more())
328 315
329 316 def test_syntax_error(self):
330 317 isp = self.isp
331 318 # Syntax errors immediately produce a 'ready' block, so the invalid
332 319 # Python can be sent to the kernel for evaluation with possible ipython
333 320 # special-syntax conversion.
334 321 isp.push('run foo')
335 322 self.assertFalse(isp.push_accepts_more())
336 323
337 324 def test_unicode(self):
338 325 self.isp.push(u"PΓ©rez")
339 326 self.isp.push(u'\xc3\xa9')
340 327 self.isp.push(u"u'\xc3\xa9'")
341 328
342 329 def test_line_continuation(self):
343 330 """ Test issue #2108."""
344 331 isp = self.isp
345 332 # A blank line after a line continuation should not accept more
346 333 isp.push("1 \\\n\n")
347 334 self.assertFalse(isp.push_accepts_more())
348 335 # Whitespace after a \ is a SyntaxError. The only way to test that
349 336 # here is to test that push doesn't accept more (as with
350 337 # test_syntax_error() above).
351 338 isp.push(r"1 \ ")
352 339 self.assertFalse(isp.push_accepts_more())
353 340 # Even if the line is continuable (c.f. the regular Python
354 341 # interpreter)
355 342 isp.push(r"(1 \ ")
356 343 self.assertFalse(isp.push_accepts_more())
357 344
358 345 class InteractiveLoopTestCase(unittest.TestCase):
359 346 """Tests for an interactive loop like a python shell.
360 347 """
361 348 def check_ns(self, lines, ns):
362 349 """Validate that the given input lines produce the resulting namespace.
363 350
364 351 Note: the input lines are given exactly as they would be typed in an
365 352 auto-indenting environment, as mini_interactive_loop above already does
366 353 auto-indenting and prepends spaces to the input.
367 354 """
368 355 src = mini_interactive_loop(pseudo_input(lines))
369 356 test_ns = {}
370 357 exec(src, test_ns)
371 358 # We can't check that the provided ns is identical to the test_ns,
372 359 # because Python fills test_ns with extra keys (copyright, etc). But
373 360 # we can check that the given dict is *contained* in test_ns
374 361 for k,v in ns.items():
375 362 self.assertEqual(test_ns[k], v)
376 363
377 364 def test_simple(self):
378 365 self.check_ns(['x=1'], dict(x=1))
379 366
380 367 def test_simple2(self):
381 368 self.check_ns(['if 1:', 'x=2'], dict(x=2))
382 369
383 370 def test_xy(self):
384 371 self.check_ns(['x=1; y=2'], dict(x=1, y=2))
385 372
386 373 def test_abc(self):
387 374 self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
388 375
389 376 def test_multi(self):
390 377 self.check_ns(['x =(1+','1+','2)'], dict(x=4))
391 378
392 379
393 380 class IPythonInputTestCase(InputSplitterTestCase):
394 381 """By just creating a new class whose .isp is a different instance, we
395 382 re-run the same test battery on the new input splitter.
396 383
397 384 In addition, this runs the tests over the syntax and syntax_ml dicts that
398 385 were tested by individual functions, as part of the OO interface.
399 386
400 387 It also makes some checks on the raw buffer storage.
401 388 """
402 389
403 390 def setUp(self):
404 391 self.isp = isp.IPythonInputSplitter()
405 392
406 393 def test_syntax(self):
407 394 """Call all single-line syntax tests from the main object"""
408 395 isp = self.isp
409 396 for example in syntax.values():
410 397 for raw, out_t in example:
411 398 if raw.startswith(' '):
412 399 continue
413 400
414 401 isp.push(raw+'\n')
415 402 out_raw = isp.source_raw
416 403 out = isp.source_reset()
417 404 self.assertEqual(out.rstrip(), out_t,
418 405 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
419 406 self.assertEqual(out_raw.rstrip(), raw.rstrip())
420 407
421 408 def test_syntax_multiline(self):
422 409 isp = self.isp
423 410 for example in syntax_ml.values():
424 411 for line_pairs in example:
425 412 out_t_parts = []
426 413 raw_parts = []
427 414 for lraw, out_t_part in line_pairs:
428 415 if out_t_part is not None:
429 416 out_t_parts.append(out_t_part)
430 417
431 418 if lraw is not None:
432 419 isp.push(lraw)
433 420 raw_parts.append(lraw)
434 421
435 422 out_raw = isp.source_raw
436 423 out = isp.source_reset()
437 424 out_t = '\n'.join(out_t_parts).rstrip()
438 425 raw = '\n'.join(raw_parts).rstrip()
439 426 self.assertEqual(out.rstrip(), out_t)
440 427 self.assertEqual(out_raw.rstrip(), raw)
441 428
442 429 def test_syntax_multiline_cell(self):
443 430 isp = self.isp
444 431 for example in syntax_ml.values():
445 432
446 433 out_t_parts = []
447 434 for line_pairs in example:
448 435 raw = '\n'.join(r for r, _ in line_pairs if r is not None)
449 436 out_t = '\n'.join(t for _,t in line_pairs if t is not None)
450 437 out = isp.transform_cell(raw)
451 438 # Match ignoring trailing whitespace
452 439 self.assertEqual(out.rstrip(), out_t.rstrip())
453 440
454 441 def test_cellmagic_preempt(self):
455 442 isp = self.isp
456 443 for raw, name, line, cell in [
457 444 ("%%cellm a\nIn[1]:", u'cellm', u'a', u'In[1]:'),
458 445 ("%%cellm \nline\n>>> hi", u'cellm', u'', u'line\n>>> hi'),
459 446 (">>> %%cellm \nline\n>>> hi", u'cellm', u'', u'line\nhi'),
460 447 ("%%cellm \n>>> hi", u'cellm', u'', u'hi'),
461 448 ("%%cellm \nline1\nline2", u'cellm', u'', u'line1\nline2'),
462 449 ("%%cellm \nline1\\\\\nline2", u'cellm', u'', u'line1\\\\\nline2'),
463 450 ]:
464 451 expected = "get_ipython().run_cell_magic(%r, %r, %r)" % (
465 452 name, line, cell
466 453 )
467 454 out = isp.transform_cell(raw)
468 455 self.assertEqual(out.rstrip(), expected.rstrip())
456
457 def test_multiline_passthrough(self):
458 isp = self.isp
459 class CommentTransformer(InputTransformer):
460 def __init__(self):
461 self._lines = []
462
463 def push(self, line):
464 self._lines.append(line + '#')
465
466 def reset(self):
467 text = '\n'.join(self._lines)
468 self._lines = []
469 return text
469 470
471 isp.physical_line_transforms.insert(0, CommentTransformer())
470 472
473 for raw, expected in [
474 ("a=5", "a=5#"),
475 ("%ls foo", "get_ipython().magic(%r)" % u'ls foo#'),
476 ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().magic(%r)" % (
477 u'ls foo#', u'ls bar#'
478 )),
479 ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().magic(%r)\n4#\n5#" % u'ls foo#'),
480 ]:
481 out = isp.transform_cell(raw)
482 self.assertEqual(out.rstrip(), expected.rstrip())
471 483
472 484 #-----------------------------------------------------------------------------
473 485 # Main - use as a script, mostly for developer experiments
474 486 #-----------------------------------------------------------------------------
475 487
476 488 if __name__ == '__main__':
477 489 # A simple demo for interactive experimentation. This code will not get
478 490 # picked up by any test suite.
479 491 from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
480 492
481 493 # configure here the syntax to use, prompt and whether to autoindent
482 494 #isp, start_prompt = InputSplitter(), '>>> '
483 495 isp, start_prompt = IPythonInputSplitter(), 'In> '
484 496
485 497 autoindent = True
486 498 #autoindent = False
487 499
488 500 try:
489 501 while True:
490 502 prompt = start_prompt
491 503 while isp.push_accepts_more():
492 504 indent = ' '*isp.indent_spaces
493 505 if autoindent:
494 506 line = indent + input(prompt+indent)
495 507 else:
496 508 line = input(prompt)
497 509 isp.push(line)
498 510 prompt = '... '
499 511
500 512 # Here we just return input so we can use it in a test suite, but a
501 513 # real interpreter would instead send it for execution somewhere.
502 514 #src = isp.source; raise EOFError # dbg
503 515 raw = isp.source_raw
504 516 src = isp.source_reset()
505 517 print('Input source was:\n', src)
506 518 print('Raw source was:\n', raw)
507 519 except EOFError:
508 520 print('Bye')
509 521
510 522 # Tests for cell magics support
511 523
512 524 def test_last_blank():
513 525 nt.assert_false(isp.last_blank(''))
514 526 nt.assert_false(isp.last_blank('abc'))
515 527 nt.assert_false(isp.last_blank('abc\n'))
516 528 nt.assert_false(isp.last_blank('abc\na'))
517 529
518 530 nt.assert_true(isp.last_blank('\n'))
519 531 nt.assert_true(isp.last_blank('\n '))
520 532 nt.assert_true(isp.last_blank('abc\n '))
521 533 nt.assert_true(isp.last_blank('abc\n\n'))
522 534 nt.assert_true(isp.last_blank('abc\nd\n\n'))
523 535 nt.assert_true(isp.last_blank('abc\nd\ne\n\n'))
524 536 nt.assert_true(isp.last_blank('abc \n \n \n\n'))
525 537
526 538
527 539 def test_last_two_blanks():
528 540 nt.assert_false(isp.last_two_blanks(''))
529 541 nt.assert_false(isp.last_two_blanks('abc'))
530 542 nt.assert_false(isp.last_two_blanks('abc\n'))
531 543 nt.assert_false(isp.last_two_blanks('abc\n\na'))
532 544 nt.assert_false(isp.last_two_blanks('abc\n \n'))
533 545 nt.assert_false(isp.last_two_blanks('abc\n\n'))
534 546
535 547 nt.assert_true(isp.last_two_blanks('\n\n'))
536 548 nt.assert_true(isp.last_two_blanks('\n\n '))
537 549 nt.assert_true(isp.last_two_blanks('\n \n'))
538 550 nt.assert_true(isp.last_two_blanks('abc\n\n '))
539 551 nt.assert_true(isp.last_two_blanks('abc\n\n\n'))
540 552 nt.assert_true(isp.last_two_blanks('abc\n\n \n'))
541 553 nt.assert_true(isp.last_two_blanks('abc\n\n \n '))
542 554 nt.assert_true(isp.last_two_blanks('abc\n\n \n \n'))
543 555 nt.assert_true(isp.last_two_blanks('abc\nd\n\n\n'))
544 556 nt.assert_true(isp.last_two_blanks('abc\nd\ne\nf\n\n\n'))
545 557
546 558
547 559 class CellMagicsCommon(object):
548 560
549 561 def test_whole_cell(self):
550 562 src = "%%cellm line\nbody\n"
551 563 out = self.sp.transform_cell(src)
552 564 ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
553 565 nt.assert_equal(out, py3compat.u_format(ref))
554 566
555 567 def test_cellmagic_help(self):
556 568 self.sp.push('%%cellm?')
557 569 nt.assert_false(self.sp.push_accepts_more())
558 570
559 571 def tearDown(self):
560 572 self.sp.reset()
561 573
562 574
563 575 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
564 576 sp = isp.IPythonInputSplitter(line_input_checker=False)
565 577
566 578 def test_incremental(self):
567 579 sp = self.sp
568 580 sp.push('%%cellm firstline\n')
569 581 nt.assert_true(sp.push_accepts_more()) #1
570 582 sp.push('line2\n')
571 583 nt.assert_true(sp.push_accepts_more()) #2
572 584 sp.push('\n')
573 585 # This should accept a blank line and carry on until the cell is reset
574 586 nt.assert_true(sp.push_accepts_more()) #3
575 587
576 588 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
577 589 sp = isp.IPythonInputSplitter(line_input_checker=True)
578 590
579 591 def test_incremental(self):
580 592 sp = self.sp
581 593 sp.push('%%cellm line2\n')
582 594 nt.assert_true(sp.push_accepts_more()) #1
583 595 sp.push('\n')
584 596 # In this case, a blank line should end the cell magic
585 597 nt.assert_false(sp.push_accepts_more()) #2
General Comments 0
You need to be logged in to leave comments. Login now