##// END OF EJS Templates
`yield`, `break`, and `continue` automatically dedent...
Aaron Meurer -
Show More
@@ -1,914 +1,918 b''
1 1 """Analysis of text input into executable blocks.
2 2
3 3 The main class in this module, :class:`InputSplitter`, is designed to break
4 4 input from either interactive, line-by-line environments or block-based ones,
5 5 into standalone blocks that can be executed by Python as 'single' statements
6 6 (thus triggering sys.displayhook).
7 7
8 8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 9 with full support for the extended IPython syntax (magics, system calls, etc).
10 10
11 11 For more details, see the class docstring below.
12 12
13 13 Syntax Transformations
14 14 ----------------------
15 15
16 16 One of the main jobs of the code in this file is to apply all syntax
17 17 transformations that make up 'the IPython language', i.e. magics, shell
18 18 escapes, etc. All transformations should be implemented as *fully stateless*
19 19 entities, that simply take one line as their input and return a line.
20 20 Internally for implementation purposes they may be a normal function or a
21 21 callable object, but the only input they receive will be a single line and they
22 22 should only return a line, without holding any data-dependent state between
23 23 calls.
24 24
25 25 As an example, the EscapedTransformer is a class so we can more clearly group
26 26 together the functionality of dispatching to individual functions based on the
27 27 starting escape character, but the only method for public use is its call
28 28 method.
29 29
30 30
31 31 ToDo
32 32 ----
33 33
34 34 - Should we make push() actually raise an exception once push_accepts_more()
35 35 returns False?
36 36
37 37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
38 38 at least just attributes of a class so not really very exposed.
39 39
40 40 - Think about the best way to support dynamic things: automagic, autocall,
41 41 macros, etc.
42 42
43 43 - Think of a better heuristic for the application of the transforms in
44 44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
45 45 track indentation change events (indent, dedent, nothing) and apply them only
46 46 if the indentation went up, but not otherwise.
47 47
48 48 - Think of the cleanest way for supporting user-specified transformations (the
49 49 user prefilters we had before).
50 50
51 51 Authors
52 52 -------
53 53
54 54 * Fernando Perez
55 55 * Brian Granger
56 56 """
57 57 #-----------------------------------------------------------------------------
58 58 # Copyright (C) 2010 The IPython Development Team
59 59 #
60 60 # Distributed under the terms of the BSD License. The full license is in
61 61 # the file COPYING, distributed as part of this software.
62 62 #-----------------------------------------------------------------------------
63 63
64 64 #-----------------------------------------------------------------------------
65 65 # Imports
66 66 #-----------------------------------------------------------------------------
67 67 # stdlib
68 68 import ast
69 69 import codeop
70 70 import re
71 71 import sys
72 72 import tokenize
73 73 from StringIO import StringIO
74 74
75 75 # IPython modules
76 76 from IPython.core.splitinput import split_user_input, LineInfo
77 77 from IPython.utils.py3compat import cast_unicode
78 78
79 79 #-----------------------------------------------------------------------------
80 80 # Globals
81 81 #-----------------------------------------------------------------------------
82 82
83 83 # The escape sequences that define the syntax transformations IPython will
84 84 # apply to user input. These can NOT be just changed here: many regular
85 85 # expressions and other parts of the code may use their hardcoded values, and
86 86 # for all intents and purposes they constitute the 'IPython syntax', so they
87 87 # should be considered fixed.
88 88
89 89 ESC_SHELL = '!' # Send line to underlying system shell
90 90 ESC_SH_CAP = '!!' # Send line to system shell and capture output
91 91 ESC_HELP = '?' # Find information about object
92 92 ESC_HELP2 = '??' # Find extra-detailed information about object
93 93 ESC_MAGIC = '%' # Call magic function
94 94 ESC_MAGIC2 = '%%' # Call cell-magic function
95 95 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
96 96 ESC_QUOTE2 = ';' # Quote all args as a single string, call
97 97 ESC_PAREN = '/' # Call first argument with rest of line as arguments
98 98
99 99 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
100 100 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
101 101 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
102 102
103 103 #-----------------------------------------------------------------------------
104 104 # Utilities
105 105 #-----------------------------------------------------------------------------
106 106
107 107 # FIXME: These are general-purpose utilities that later can be moved to the
108 108 # general ward. Kept here for now because we're being very strict about test
109 109 # coverage with this code, and this lets us ensure that we keep 100% coverage
110 110 # while developing.
111 111
112 112 # compiled regexps for autoindent management
113 113 dedent_re = re.compile('|'.join([
114 114 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
115 115 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
116 116 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
117 117 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
118 r'^\s+pass\s*$' # pass (optionally followed by trailing spaces)
118 r'^\s+yield(\s.*)?$', # normal yield (+ space + other stuff, maybe)
119 r'^\s+yield\([^\)]*\).*$', # wacky yield with immediate open paren
120 r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
121 r'^\s+break\s*$', # break (optionally followed by trailing spaces)
122 r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
119 123 ]))
120 124 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
121 125
122 126 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
123 127 # before pure comments
124 128 comment_line_re = re.compile('^\s*\#')
125 129
126 130
127 131 def num_ini_spaces(s):
128 132 """Return the number of initial spaces in a string.
129 133
130 134 Note that tabs are counted as a single space. For now, we do *not* support
131 135 mixing of tabs and spaces in the user's input.
132 136
133 137 Parameters
134 138 ----------
135 139 s : string
136 140
137 141 Returns
138 142 -------
139 143 n : int
140 144 """
141 145
142 146 ini_spaces = ini_spaces_re.match(s)
143 147 if ini_spaces:
144 148 return ini_spaces.end()
145 149 else:
146 150 return 0
147 151
148 152 def last_blank(src):
149 153 """Determine if the input source ends in a blank.
150 154
151 155 A blank is either a newline or a line consisting of whitespace.
152 156
153 157 Parameters
154 158 ----------
155 159 src : string
156 160 A single or multiline string.
157 161 """
158 162 if not src: return False
159 163 ll = src.splitlines()[-1]
160 164 return (ll == '') or ll.isspace()
161 165
162 166
163 167 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
164 168 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
165 169
166 170 def last_two_blanks(src):
167 171 """Determine if the input source ends in two blanks.
168 172
169 173 A blank is either a newline or a line consisting of whitespace.
170 174
171 175 Parameters
172 176 ----------
173 177 src : string
174 178 A single or multiline string.
175 179 """
176 180 if not src: return False
177 181 # The logic here is tricky: I couldn't get a regexp to work and pass all
178 182 # the tests, so I took a different approach: split the source by lines,
179 183 # grab the last two and prepend '###\n' as a stand-in for whatever was in
180 184 # the body before the last two lines. Then, with that structure, it's
181 185 # possible to analyze with two regexps. Not the most elegant solution, but
182 186 # it works. If anyone tries to change this logic, make sure to validate
183 187 # the whole test suite first!
184 188 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
185 189 return (bool(last_two_blanks_re.match(new_src)) or
186 190 bool(last_two_blanks_re2.match(new_src)) )
187 191
188 192
189 193 def remove_comments(src):
190 194 """Remove all comments from input source.
191 195
192 196 Note: comments are NOT recognized inside of strings!
193 197
194 198 Parameters
195 199 ----------
196 200 src : string
197 201 A single or multiline input string.
198 202
199 203 Returns
200 204 -------
201 205 String with all Python comments removed.
202 206 """
203 207
204 208 return re.sub('#.*', '', src)
205 209
206 210 def has_comment(src):
207 211 """Indicate whether an input line has (i.e. ends in, or is) a comment.
208 212
209 213 This uses tokenize, so it can distinguish comments from # inside strings.
210 214
211 215 Parameters
212 216 ----------
213 217 src : string
214 218 A single line input string.
215 219
216 220 Returns
217 221 -------
218 222 Boolean: True if source has a comment.
219 223 """
220 224 readline = StringIO(src).readline
221 225 toktypes = set()
222 226 try:
223 227 for t in tokenize.generate_tokens(readline):
224 228 toktypes.add(t[0])
225 229 except tokenize.TokenError:
226 230 pass
227 231 return(tokenize.COMMENT in toktypes)
228 232
229 233
230 234 def get_input_encoding():
231 235 """Return the default standard input encoding.
232 236
233 237 If sys.stdin has no encoding, 'ascii' is returned."""
234 238 # There are strange environments for which sys.stdin.encoding is None. We
235 239 # ensure that a valid encoding is returned.
236 240 encoding = getattr(sys.stdin, 'encoding', None)
237 241 if encoding is None:
238 242 encoding = 'ascii'
239 243 return encoding
240 244
241 245 #-----------------------------------------------------------------------------
242 246 # Classes and functions for normal Python syntax handling
243 247 #-----------------------------------------------------------------------------
244 248
245 249 class InputSplitter(object):
246 250 """An object that can accumulate lines of Python source before execution.
247 251
248 252 This object is designed to be fed python source line-by-line, using
249 253 :meth:`push`. It will return on each push whether the currently pushed
250 254 code could be executed already. In addition, it provides a method called
251 255 :meth:`push_accepts_more` that can be used to query whether more input
252 256 can be pushed into a single interactive block.
253 257
254 258 This is a simple example of how an interactive terminal-based client can use
255 259 this tool::
256 260
257 261 isp = InputSplitter()
258 262 while isp.push_accepts_more():
259 263 indent = ' '*isp.indent_spaces
260 264 prompt = '>>> ' + indent
261 265 line = indent + raw_input(prompt)
262 266 isp.push(line)
263 267 print 'Input source was:\n', isp.source_reset(),
264 268 """
265 269 # Number of spaces of indentation computed from input that has been pushed
266 270 # so far. This is the attributes callers should query to get the current
267 271 # indentation level, in order to provide auto-indent facilities.
268 272 indent_spaces = 0
269 273 # String, indicating the default input encoding. It is computed by default
270 274 # at initialization time via get_input_encoding(), but it can be reset by a
271 275 # client with specific knowledge of the encoding.
272 276 encoding = ''
273 277 # String where the current full source input is stored, properly encoded.
274 278 # Reading this attribute is the normal way of querying the currently pushed
275 279 # source code, that has been properly encoded.
276 280 source = ''
277 281 # Code object corresponding to the current source. It is automatically
278 282 # synced to the source, so it can be queried at any time to obtain the code
279 283 # object; it will be None if the source doesn't compile to valid Python.
280 284 code = None
281 285 # Input mode
282 286 input_mode = 'line'
283 287
284 288 # Private attributes
285 289
286 290 # List with lines of input accumulated so far
287 291 _buffer = None
288 292 # Command compiler
289 293 _compile = None
290 294 # Mark when input has changed indentation all the way back to flush-left
291 295 _full_dedent = False
292 296 # Boolean indicating whether the current block is complete
293 297 _is_complete = None
294 298
295 299 def __init__(self, input_mode=None):
296 300 """Create a new InputSplitter instance.
297 301
298 302 Parameters
299 303 ----------
300 304 input_mode : str
301 305
302 306 One of ['line', 'cell']; default is 'line'.
303 307
304 308 The input_mode parameter controls how new inputs are used when fed via
305 309 the :meth:`push` method:
306 310
307 311 - 'line': meant for line-oriented clients, inputs are appended one at a
308 312 time to the internal buffer and the whole buffer is compiled.
309 313
310 314 - 'cell': meant for clients that can edit multi-line 'cells' of text at
311 315 a time. A cell can contain one or more blocks that can be compile in
312 316 'single' mode by Python. In this mode, each new input new input
313 317 completely replaces all prior inputs. Cell mode is thus equivalent
314 318 to prepending a full reset() to every push() call.
315 319 """
316 320 self._buffer = []
317 321 self._compile = codeop.CommandCompiler()
318 322 self.encoding = get_input_encoding()
319 323 self.input_mode = InputSplitter.input_mode if input_mode is None \
320 324 else input_mode
321 325
322 326 def reset(self):
323 327 """Reset the input buffer and associated state."""
324 328 self.indent_spaces = 0
325 329 self._buffer[:] = []
326 330 self.source = ''
327 331 self.code = None
328 332 self._is_complete = False
329 333 self._full_dedent = False
330 334
331 335 def source_reset(self):
332 336 """Return the input source and perform a full reset.
333 337 """
334 338 out = self.source
335 339 self.reset()
336 340 return out
337 341
338 342 def push(self, lines):
339 343 """Push one or more lines of input.
340 344
341 345 This stores the given lines and returns a status code indicating
342 346 whether the code forms a complete Python block or not.
343 347
344 348 Any exceptions generated in compilation are swallowed, but if an
345 349 exception was produced, the method returns True.
346 350
347 351 Parameters
348 352 ----------
349 353 lines : string
350 354 One or more lines of Python input.
351 355
352 356 Returns
353 357 -------
354 358 is_complete : boolean
355 359 True if the current input source (the result of the current input
356 360 plus prior inputs) forms a complete Python execution block. Note that
357 361 this value is also stored as a private attribute (``_is_complete``), so it
358 362 can be queried at any time.
359 363 """
360 364 if self.input_mode == 'cell':
361 365 self.reset()
362 366
363 367 self._store(lines)
364 368 source = self.source
365 369
366 370 # Before calling _compile(), reset the code object to None so that if an
367 371 # exception is raised in compilation, we don't mislead by having
368 372 # inconsistent code/source attributes.
369 373 self.code, self._is_complete = None, None
370 374
371 375 # Honor termination lines properly
372 376 if source.endswith('\\\n'):
373 377 return False
374 378
375 379 self._update_indent(lines)
376 380 try:
377 381 self.code = self._compile(source, symbol="exec")
378 382 # Invalid syntax can produce any of a number of different errors from
379 383 # inside the compiler, so we have to catch them all. Syntax errors
380 384 # immediately produce a 'ready' block, so the invalid Python can be
381 385 # sent to the kernel for evaluation with possible ipython
382 386 # special-syntax conversion.
383 387 except (SyntaxError, OverflowError, ValueError, TypeError,
384 388 MemoryError):
385 389 self._is_complete = True
386 390 else:
387 391 # Compilation didn't produce any exceptions (though it may not have
388 392 # given a complete code object)
389 393 self._is_complete = self.code is not None
390 394
391 395 return self._is_complete
392 396
393 397 def push_accepts_more(self):
394 398 """Return whether a block of interactive input can accept more input.
395 399
396 400 This method is meant to be used by line-oriented frontends, who need to
397 401 guess whether a block is complete or not based solely on prior and
398 402 current input lines. The InputSplitter considers it has a complete
399 403 interactive block and will not accept more input only when either a
400 404 SyntaxError is raised, or *all* of the following are true:
401 405
402 406 1. The input compiles to a complete statement.
403 407
404 408 2. The indentation level is flush-left (because if we are indented,
405 409 like inside a function definition or for loop, we need to keep
406 410 reading new input).
407 411
408 412 3. There is one extra line consisting only of whitespace.
409 413
410 414 Because of condition #3, this method should be used only by
411 415 *line-oriented* frontends, since it means that intermediate blank lines
412 416 are not allowed in function definitions (or any other indented block).
413 417
414 418 If the current input produces a syntax error, this method immediately
415 419 returns False but does *not* raise the syntax error exception, as
416 420 typically clients will want to send invalid syntax to an execution
417 421 backend which might convert the invalid syntax into valid Python via
418 422 one of the dynamic IPython mechanisms.
419 423 """
420 424
421 425 # With incomplete input, unconditionally accept more
422 426 if not self._is_complete:
423 427 return True
424 428
425 429 # If we already have complete input and we're flush left, the answer
426 430 # depends. In line mode, if there hasn't been any indentation,
427 431 # that's it. If we've come back from some indentation, we need
428 432 # the blank final line to finish.
429 433 # In cell mode, we need to check how many blocks the input so far
430 434 # compiles into, because if there's already more than one full
431 435 # independent block of input, then the client has entered full
432 436 # 'cell' mode and is feeding lines that each is complete. In this
433 437 # case we should then keep accepting. The Qt terminal-like console
434 438 # does precisely this, to provide the convenience of terminal-like
435 439 # input of single expressions, but allowing the user (with a
436 440 # separate keystroke) to switch to 'cell' mode and type multiple
437 441 # expressions in one shot.
438 442 if self.indent_spaces==0:
439 443 if self.input_mode=='line':
440 444 if not self._full_dedent:
441 445 return False
442 446 else:
443 447 try:
444 448 code_ast = ast.parse(u''.join(self._buffer))
445 449 except Exception:
446 450 return False
447 451 else:
448 452 if len(code_ast.body) == 1:
449 453 return False
450 454
451 455 # When input is complete, then termination is marked by an extra blank
452 456 # line at the end.
453 457 last_line = self.source.splitlines()[-1]
454 458 return bool(last_line and not last_line.isspace())
455 459
456 460 #------------------------------------------------------------------------
457 461 # Private interface
458 462 #------------------------------------------------------------------------
459 463
460 464 def _find_indent(self, line):
461 465 """Compute the new indentation level for a single line.
462 466
463 467 Parameters
464 468 ----------
465 469 line : str
466 470 A single new line of non-whitespace, non-comment Python input.
467 471
468 472 Returns
469 473 -------
470 474 indent_spaces : int
471 475 New value for the indent level (it may be equal to self.indent_spaces
472 476 if indentation doesn't change.
473 477
474 478 full_dedent : boolean
475 479 Whether the new line causes a full flush-left dedent.
476 480 """
477 481 indent_spaces = self.indent_spaces
478 482 full_dedent = self._full_dedent
479 483
480 484 inisp = num_ini_spaces(line)
481 485 if inisp < indent_spaces:
482 486 indent_spaces = inisp
483 487 if indent_spaces <= 0:
484 488 #print 'Full dedent in text',self.source # dbg
485 489 full_dedent = True
486 490
487 491 if line.rstrip()[-1] == ':':
488 492 indent_spaces += 4
489 493 elif dedent_re.match(line):
490 494 indent_spaces -= 4
491 495 if indent_spaces <= 0:
492 496 full_dedent = True
493 497
494 498 # Safety
495 499 if indent_spaces < 0:
496 500 indent_spaces = 0
497 501 #print 'safety' # dbg
498 502
499 503 return indent_spaces, full_dedent
500 504
501 505 def _update_indent(self, lines):
502 506 for line in remove_comments(lines).splitlines():
503 507 if line and not line.isspace():
504 508 self.indent_spaces, self._full_dedent = self._find_indent(line)
505 509
506 510 def _store(self, lines, buffer=None, store='source'):
507 511 """Store one or more lines of input.
508 512
509 513 If input lines are not newline-terminated, a newline is automatically
510 514 appended."""
511 515
512 516 if buffer is None:
513 517 buffer = self._buffer
514 518
515 519 if lines.endswith('\n'):
516 520 buffer.append(lines)
517 521 else:
518 522 buffer.append(lines+'\n')
519 523 setattr(self, store, self._set_source(buffer))
520 524
521 525 def _set_source(self, buffer):
522 526 return u''.join(buffer)
523 527
524 528
525 529 #-----------------------------------------------------------------------------
526 530 # Functions and classes for IPython-specific syntactic support
527 531 #-----------------------------------------------------------------------------
528 532
529 533 # The escaped translators ALL receive a line where their own escape has been
530 534 # stripped. Only '?' is valid at the end of the line, all others can only be
531 535 # placed at the start.
532 536
533 537 # Transformations of the special syntaxes that don't rely on an explicit escape
534 538 # character but instead on patterns on the input line
535 539
536 540 # The core transformations are implemented as standalone functions that can be
537 541 # tested and validated in isolation. Each of these uses a regexp, we
538 542 # pre-compile these and keep them close to each function definition for clarity
539 543
540 544 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
541 545 r'\s*=\s*!\s*(?P<cmd>.*)')
542 546
543 547 def transform_assign_system(line):
544 548 """Handle the `files = !ls` syntax."""
545 549 m = _assign_system_re.match(line)
546 550 if m is not None:
547 551 cmd = m.group('cmd')
548 552 lhs = m.group('lhs')
549 553 new_line = '%s = get_ipython().getoutput(%r)' % (lhs, cmd)
550 554 return new_line
551 555 return line
552 556
553 557
554 558 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
555 559 r'\s*=\s*%\s*(?P<cmd>.*)')
556 560
557 561 def transform_assign_magic(line):
558 562 """Handle the `a = %who` syntax."""
559 563 m = _assign_magic_re.match(line)
560 564 if m is not None:
561 565 cmd = m.group('cmd')
562 566 lhs = m.group('lhs')
563 567 new_line = '%s = get_ipython().magic(%r)' % (lhs, cmd)
564 568 return new_line
565 569 return line
566 570
567 571
568 572 _classic_prompt_re = re.compile(r'^([ \t]*>>> |^[ \t]*\.\.\. )')
569 573
570 574 def transform_classic_prompt(line):
571 575 """Handle inputs that start with '>>> ' syntax."""
572 576
573 577 if not line or line.isspace():
574 578 return line
575 579 m = _classic_prompt_re.match(line)
576 580 if m:
577 581 return line[len(m.group(0)):]
578 582 else:
579 583 return line
580 584
581 585
582 586 _ipy_prompt_re = re.compile(r'^([ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
583 587
584 588 def transform_ipy_prompt(line):
585 589 """Handle inputs that start classic IPython prompt syntax."""
586 590
587 591 if not line or line.isspace():
588 592 return line
589 593 #print 'LINE: %r' % line # dbg
590 594 m = _ipy_prompt_re.match(line)
591 595 if m:
592 596 #print 'MATCH! %r -> %r' % (line, line[len(m.group(0)):]) # dbg
593 597 return line[len(m.group(0)):]
594 598 else:
595 599 return line
596 600
597 601
598 602 def _make_help_call(target, esc, lspace, next_input=None):
599 603 """Prepares a pinfo(2)/psearch call from a target name and the escape
600 604 (i.e. ? or ??)"""
601 605 method = 'pinfo2' if esc == '??' \
602 606 else 'psearch' if '*' in target \
603 607 else 'pinfo'
604 608 arg = " ".join([method, target])
605 609 if next_input is None:
606 610 return '%sget_ipython().magic(%r)' % (lspace, arg)
607 611 else:
608 612 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
609 613 (lspace, next_input, arg)
610 614
611 615
612 616 _initial_space_re = re.compile(r'\s*')
613 617
614 618 _help_end_re = re.compile(r"""(%{0,2}
615 619 [a-zA-Z_*][\w*]* # Variable name
616 620 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
617 621 )
618 622 (\?\??)$ # ? or ??""",
619 623 re.VERBOSE)
620 624
621 625
622 626 def transform_help_end(line):
623 627 """Translate lines with ?/?? at the end"""
624 628 m = _help_end_re.search(line)
625 629 if m is None or has_comment(line):
626 630 return line
627 631 target = m.group(1)
628 632 esc = m.group(3)
629 633 lspace = _initial_space_re.match(line).group(0)
630 634
631 635 # If we're mid-command, put it back on the next prompt for the user.
632 636 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
633 637
634 638 return _make_help_call(target, esc, lspace, next_input)
635 639
636 640
637 641 class EscapedTransformer(object):
638 642 """Class to transform lines that are explicitly escaped out."""
639 643
640 644 def __init__(self):
641 645 tr = { ESC_SHELL : self._tr_system,
642 646 ESC_SH_CAP : self._tr_system2,
643 647 ESC_HELP : self._tr_help,
644 648 ESC_HELP2 : self._tr_help,
645 649 ESC_MAGIC : self._tr_magic,
646 650 ESC_QUOTE : self._tr_quote,
647 651 ESC_QUOTE2 : self._tr_quote2,
648 652 ESC_PAREN : self._tr_paren }
649 653 self.tr = tr
650 654
651 655 # Support for syntax transformations that use explicit escapes typed by the
652 656 # user at the beginning of a line
653 657 @staticmethod
654 658 def _tr_system(line_info):
655 659 "Translate lines escaped with: !"
656 660 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
657 661 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
658 662
659 663 @staticmethod
660 664 def _tr_system2(line_info):
661 665 "Translate lines escaped with: !!"
662 666 cmd = line_info.line.lstrip()[2:]
663 667 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
664 668
665 669 @staticmethod
666 670 def _tr_help(line_info):
667 671 "Translate lines escaped with: ?/??"
668 672 # A naked help line should just fire the intro help screen
669 673 if not line_info.line[1:]:
670 674 return 'get_ipython().show_usage()'
671 675
672 676 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
673 677
674 678 @staticmethod
675 679 def _tr_magic(line_info):
676 680 "Translate lines escaped with: %"
677 681 tpl = '%sget_ipython().magic(%r)'
678 682 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
679 683 return tpl % (line_info.pre, cmd)
680 684
681 685 @staticmethod
682 686 def _tr_quote(line_info):
683 687 "Translate lines escaped with: ,"
684 688 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
685 689 '", "'.join(line_info.the_rest.split()) )
686 690
687 691 @staticmethod
688 692 def _tr_quote2(line_info):
689 693 "Translate lines escaped with: ;"
690 694 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
691 695 line_info.the_rest)
692 696
693 697 @staticmethod
694 698 def _tr_paren(line_info):
695 699 "Translate lines escaped with: /"
696 700 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
697 701 ", ".join(line_info.the_rest.split()))
698 702
699 703 def __call__(self, line):
700 704 """Class to transform lines that are explicitly escaped out.
701 705
702 706 This calls the above _tr_* static methods for the actual line
703 707 translations."""
704 708
705 709 # Empty lines just get returned unmodified
706 710 if not line or line.isspace():
707 711 return line
708 712
709 713 # Get line endpoints, where the escapes can be
710 714 line_info = LineInfo(line)
711 715
712 716 if not line_info.esc in self.tr:
713 717 # If we don't recognize the escape, don't modify the line
714 718 return line
715 719
716 720 return self.tr[line_info.esc](line_info)
717 721
718 722
719 723 # A function-looking object to be used by the rest of the code. The purpose of
720 724 # the class in this case is to organize related functionality, more than to
721 725 # manage state.
722 726 transform_escaped = EscapedTransformer()
723 727
724 728
725 729 class IPythonInputSplitter(InputSplitter):
726 730 """An input splitter that recognizes all of IPython's special syntax."""
727 731
728 732 # String with raw, untransformed input.
729 733 source_raw = ''
730 734
731 735 # Flag to track when we're in the middle of processing a cell magic, since
732 736 # the logic has to change. In that case, we apply no transformations at
733 737 # all.
734 738 processing_cell_magic = False
735 739
736 740 # Storage for all blocks of input that make up a cell magic
737 741 cell_magic_parts = []
738 742
739 743 # Private attributes
740 744
741 745 # List with lines of raw input accumulated so far.
742 746 _buffer_raw = None
743 747
744 748 def __init__(self, input_mode=None):
745 749 super(IPythonInputSplitter, self).__init__(input_mode)
746 750 self._buffer_raw = []
747 751 self._validate = True
748 752
749 753 def reset(self):
750 754 """Reset the input buffer and associated state."""
751 755 super(IPythonInputSplitter, self).reset()
752 756 self._buffer_raw[:] = []
753 757 self.source_raw = ''
754 758 self.cell_magic_parts = []
755 759 self.processing_cell_magic = False
756 760
757 761 def source_raw_reset(self):
758 762 """Return input and raw source and perform a full reset.
759 763 """
760 764 out = self.source
761 765 out_r = self.source_raw
762 766 self.reset()
763 767 return out, out_r
764 768
765 769 def push_accepts_more(self):
766 770 if self.processing_cell_magic:
767 771 return not self._is_complete
768 772 else:
769 773 return super(IPythonInputSplitter, self).push_accepts_more()
770 774
771 775 def _handle_cell_magic(self, lines):
772 776 """Process lines when they start with %%, which marks cell magics.
773 777 """
774 778 self.processing_cell_magic = True
775 779 first, _, body = lines.partition('\n')
776 780 magic_name, _, line = first.partition(' ')
777 781 magic_name = magic_name.lstrip(ESC_MAGIC)
778 782 # We store the body of the cell and create a call to a method that
779 783 # will use this stored value. This is ugly, but it's a first cut to
780 784 # get it all working, as right now changing the return API of our
781 785 # methods would require major refactoring.
782 786 self.cell_magic_parts = [body]
783 787 tpl = 'get_ipython()._run_cached_cell_magic(%r, %r)'
784 788 tlines = tpl % (magic_name, line)
785 789 self._store(tlines)
786 790 self._store(lines, self._buffer_raw, 'source_raw')
787 791 # We can actually choose whether to allow for single blank lines here
788 792 # during input for clients that use cell mode to decide when to stop
789 793 # pushing input (currently only the Qt console).
790 794 # My first implementation did that, and then I realized it wasn't
791 795 # consistent with the terminal behavior, so I've reverted it to one
792 796 # line. But I'm leaving it here so we can easily test both behaviors,
793 797 # I kind of liked having full blank lines allowed in the cell magics...
794 798 #self._is_complete = last_two_blanks(lines)
795 799 self._is_complete = last_blank(lines)
796 800 return self._is_complete
797 801
798 802 def _line_mode_cell_append(self, lines):
799 803 """Append new content for a cell magic in line mode.
800 804 """
801 805 # Only store the raw input. Lines beyond the first one are only only
802 806 # stored for history purposes; for execution the caller will grab the
803 807 # magic pieces from cell_magic_parts and will assemble the cell body
804 808 self._store(lines, self._buffer_raw, 'source_raw')
805 809 self.cell_magic_parts.append(lines)
806 810 # Find out if the last stored block has a whitespace line as its
807 811 # last line and also this line is whitespace, case in which we're
808 812 # done (two contiguous blank lines signal termination). Note that
809 813 # the storage logic *enforces* that every stored block is
810 814 # newline-terminated, so we grab everything but the last character
811 815 # so we can have the body of the block alone.
812 816 last_block = self.cell_magic_parts[-1]
813 817 self._is_complete = last_blank(last_block) and lines.isspace()
814 818 return self._is_complete
815 819
816 820 def transform_cell(self, cell):
817 821 """Process and translate a cell of input.
818 822 """
819 823 self.reset()
820 824 self.push(cell)
821 825 return self.source_reset()
822 826
823 827 def push(self, lines):
824 828 """Push one or more lines of IPython input.
825 829
826 830 This stores the given lines and returns a status code indicating
827 831 whether the code forms a complete Python block or not, after processing
828 832 all input lines for special IPython syntax.
829 833
830 834 Any exceptions generated in compilation are swallowed, but if an
831 835 exception was produced, the method returns True.
832 836
833 837 Parameters
834 838 ----------
835 839 lines : string
836 840 One or more lines of Python input.
837 841
838 842 Returns
839 843 -------
840 844 is_complete : boolean
841 845 True if the current input source (the result of the current input
842 846 plus prior inputs) forms a complete Python execution block. Note that
843 847 this value is also stored as a private attribute (_is_complete), so it
844 848 can be queried at any time.
845 849 """
846 850 if not lines:
847 851 return super(IPythonInputSplitter, self).push(lines)
848 852
849 853 # We must ensure all input is pure unicode
850 854 lines = cast_unicode(lines, self.encoding)
851 855
852 856 # If the entire input block is a cell magic, return after handling it
853 857 # as the rest of the transformation logic should be skipped.
854 858 if lines.startswith('%%') and not \
855 859 (len(lines.splitlines()) == 1 and lines.strip().endswith('?')):
856 860 return self._handle_cell_magic(lines)
857 861
858 862 # In line mode, a cell magic can arrive in separate pieces
859 863 if self.input_mode == 'line' and self.processing_cell_magic:
860 864 return self._line_mode_cell_append(lines)
861 865
862 866 # The rest of the processing is for 'normal' content, i.e. IPython
863 867 # source that we process through our transformations pipeline.
864 868 lines_list = lines.splitlines()
865 869
866 870 transforms = [transform_ipy_prompt, transform_classic_prompt,
867 871 transform_help_end, transform_escaped,
868 872 transform_assign_system, transform_assign_magic]
869 873
870 874 # Transform logic
871 875 #
872 876 # We only apply the line transformers to the input if we have either no
873 877 # input yet, or complete input, or if the last line of the buffer ends
874 878 # with ':' (opening an indented block). This prevents the accidental
875 879 # transformation of escapes inside multiline expressions like
876 880 # triple-quoted strings or parenthesized expressions.
877 881 #
878 882 # The last heuristic, while ugly, ensures that the first line of an
879 883 # indented block is correctly transformed.
880 884 #
881 885 # FIXME: try to find a cleaner approach for this last bit.
882 886
883 887 # If we were in 'block' mode, since we're going to pump the parent
884 888 # class by hand line by line, we need to temporarily switch out to
885 889 # 'line' mode, do a single manual reset and then feed the lines one
886 890 # by one. Note that this only matters if the input has more than one
887 891 # line.
888 892 changed_input_mode = False
889 893
890 894 if self.input_mode == 'cell':
891 895 self.reset()
892 896 changed_input_mode = True
893 897 saved_input_mode = 'cell'
894 898 self.input_mode = 'line'
895 899
896 900 # Store raw source before applying any transformations to it. Note
897 901 # that this must be done *after* the reset() call that would otherwise
898 902 # flush the buffer.
899 903 self._store(lines, self._buffer_raw, 'source_raw')
900 904
901 905 try:
902 906 push = super(IPythonInputSplitter, self).push
903 907 buf = self._buffer
904 908 for line in lines_list:
905 909 if self._is_complete or not buf or \
906 910 (buf and buf[-1].rstrip().endswith((':', ','))):
907 911 for f in transforms:
908 912 line = f(line)
909 913
910 914 out = push(line)
911 915 finally:
912 916 if changed_input_mode:
913 917 self.input_mode = saved_input_mode
914 918 return out
@@ -1,825 +1,859 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Tests for the inputsplitter module.
3 3
4 4 Authors
5 5 -------
6 6 * Fernando Perez
7 7 * Robert Kern
8 8 """
9 9 #-----------------------------------------------------------------------------
10 10 # Copyright (C) 2010-2011 The IPython Development Team
11 11 #
12 12 # Distributed under the terms of the BSD License. The full license is in
13 13 # the file COPYING, distributed as part of this software.
14 14 #-----------------------------------------------------------------------------
15 15
16 16 #-----------------------------------------------------------------------------
17 17 # Imports
18 18 #-----------------------------------------------------------------------------
19 19 # stdlib
20 20 import unittest
21 21 import sys
22 22
23 23 # Third party
24 24 import nose.tools as nt
25 25
26 26 # Our own
27 27 from IPython.core import inputsplitter as isp
28 28 from IPython.testing import tools as tt
29 29 from IPython.utils import py3compat
30 30
31 31 #-----------------------------------------------------------------------------
32 32 # Semi-complete examples (also used as tests)
33 33 #-----------------------------------------------------------------------------
34 34
35 35 # Note: at the bottom, there's a slightly more complete version of this that
36 36 # can be useful during development of code here.
37 37
38 38 def mini_interactive_loop(input_func):
39 39 """Minimal example of the logic of an interactive interpreter loop.
40 40
41 41 This serves as an example, and it is used by the test system with a fake
42 42 raw_input that simulates interactive input."""
43 43
44 44 from IPython.core.inputsplitter import InputSplitter
45 45
46 46 isp = InputSplitter()
47 47 # In practice, this input loop would be wrapped in an outside loop to read
48 48 # input indefinitely, until some exit/quit command was issued. Here we
49 49 # only illustrate the basic inner loop.
50 50 while isp.push_accepts_more():
51 51 indent = ' '*isp.indent_spaces
52 52 prompt = '>>> ' + indent
53 53 line = indent + input_func(prompt)
54 54 isp.push(line)
55 55
56 56 # Here we just return input so we can use it in a test suite, but a real
57 57 # interpreter would instead send it for execution somewhere.
58 58 src = isp.source_reset()
59 59 #print 'Input source was:\n', src # dbg
60 60 return src
61 61
62 62 #-----------------------------------------------------------------------------
63 63 # Test utilities, just for local use
64 64 #-----------------------------------------------------------------------------
65 65
66 66 def assemble(block):
67 67 """Assemble a block into multi-line sub-blocks."""
68 68 return ['\n'.join(sub_block)+'\n' for sub_block in block]
69 69
70 70
71 71 def pseudo_input(lines):
72 72 """Return a function that acts like raw_input but feeds the input list."""
73 73 ilines = iter(lines)
74 74 def raw_in(prompt):
75 75 try:
76 76 return next(ilines)
77 77 except StopIteration:
78 78 return ''
79 79 return raw_in
80 80
81 81 #-----------------------------------------------------------------------------
82 82 # Tests
83 83 #-----------------------------------------------------------------------------
84 84 def test_spaces():
85 85 tests = [('', 0),
86 86 (' ', 1),
87 87 ('\n', 0),
88 88 (' \n', 1),
89 89 ('x', 0),
90 90 (' x', 1),
91 91 (' x',2),
92 92 (' x',4),
93 93 # Note: tabs are counted as a single whitespace!
94 94 ('\tx', 1),
95 95 ('\t x', 2),
96 96 ]
97 97 tt.check_pairs(isp.num_ini_spaces, tests)
98 98
99 99
100 100 def test_remove_comments():
101 101 tests = [('text', 'text'),
102 102 ('text # comment', 'text '),
103 103 ('text # comment\n', 'text \n'),
104 104 ('text # comment \n', 'text \n'),
105 105 ('line # c \nline\n','line \nline\n'),
106 106 ('line # c \nline#c2 \nline\nline #c\n\n',
107 107 'line \nline\nline\nline \n\n'),
108 108 ]
109 109 tt.check_pairs(isp.remove_comments, tests)
110 110
111 111 def test_has_comment():
112 112 tests = [('text', False),
113 113 ('text #comment', True),
114 114 ('text #comment\n', True),
115 115 ('#comment', True),
116 116 ('#comment\n', True),
117 117 ('a = "#string"', False),
118 118 ('a = "#string" # comment', True),
119 119 ('a #comment not "string"', True),
120 120 ]
121 121 tt.check_pairs(isp.has_comment, tests)
122 122
123 123
124 124 def test_get_input_encoding():
125 125 encoding = isp.get_input_encoding()
126 126 nt.assert_true(isinstance(encoding, basestring))
127 127 # simple-minded check that at least encoding a simple string works with the
128 128 # encoding we got.
129 129 nt.assert_equal(u'test'.encode(encoding), b'test')
130 130
131 131
132 132 class NoInputEncodingTestCase(unittest.TestCase):
133 133 def setUp(self):
134 134 self.old_stdin = sys.stdin
135 135 class X: pass
136 136 fake_stdin = X()
137 137 sys.stdin = fake_stdin
138 138
139 139 def test(self):
140 140 # Verify that if sys.stdin has no 'encoding' attribute we do the right
141 141 # thing
142 142 enc = isp.get_input_encoding()
143 143 self.assertEqual(enc, 'ascii')
144 144
145 145 def tearDown(self):
146 146 sys.stdin = self.old_stdin
147 147
148 148
149 149 class InputSplitterTestCase(unittest.TestCase):
150 150 def setUp(self):
151 151 self.isp = isp.InputSplitter()
152 152
153 153 def test_reset(self):
154 154 isp = self.isp
155 155 isp.push('x=1')
156 156 isp.reset()
157 157 self.assertEqual(isp._buffer, [])
158 158 self.assertEqual(isp.indent_spaces, 0)
159 159 self.assertEqual(isp.source, '')
160 160 self.assertEqual(isp.code, None)
161 161 self.assertEqual(isp._is_complete, False)
162 162
163 163 def test_source(self):
164 164 self.isp._store('1')
165 165 self.isp._store('2')
166 166 self.assertEqual(self.isp.source, '1\n2\n')
167 167 self.assertTrue(len(self.isp._buffer)>0)
168 168 self.assertEqual(self.isp.source_reset(), '1\n2\n')
169 169 self.assertEqual(self.isp._buffer, [])
170 170 self.assertEqual(self.isp.source, '')
171 171
172 172 def test_indent(self):
173 173 isp = self.isp # shorthand
174 174 isp.push('x=1')
175 175 self.assertEqual(isp.indent_spaces, 0)
176 176 isp.push('if 1:\n x=1')
177 177 self.assertEqual(isp.indent_spaces, 4)
178 178 isp.push('y=2\n')
179 179 self.assertEqual(isp.indent_spaces, 0)
180 180
181 181 def test_indent2(self):
182 182 # In cell mode, inputs must be fed in whole blocks, so skip this test
183 183 if self.isp.input_mode == 'cell': return
184 184
185 185 isp = self.isp
186 186 isp.push('if 1:')
187 187 self.assertEqual(isp.indent_spaces, 4)
188 188 isp.push(' x=1')
189 189 self.assertEqual(isp.indent_spaces, 4)
190 190 # Blank lines shouldn't change the indent level
191 191 isp.push(' '*2)
192 192 self.assertEqual(isp.indent_spaces, 4)
193 193
194 194 def test_indent3(self):
195 195 # In cell mode, inputs must be fed in whole blocks, so skip this test
196 196 if self.isp.input_mode == 'cell': return
197 197
198 198 isp = self.isp
199 199 # When a multiline statement contains parens or multiline strings, we
200 200 # shouldn't get confused.
201 201 isp.push("if 1:")
202 202 isp.push(" x = (1+\n 2)")
203 203 self.assertEqual(isp.indent_spaces, 4)
204 204
205 205 def test_indent4(self):
206 206 # In cell mode, inputs must be fed in whole blocks, so skip this test
207 207 if self.isp.input_mode == 'cell': return
208 208
209 209 isp = self.isp
210 210 # whitespace after ':' should not screw up indent level
211 211 isp.push('if 1: \n x=1')
212 212 self.assertEqual(isp.indent_spaces, 4)
213 213 isp.push('y=2\n')
214 214 self.assertEqual(isp.indent_spaces, 0)
215 215 isp.push('if 1:\t\n x=1')
216 216 self.assertEqual(isp.indent_spaces, 4)
217 217 isp.push('y=2\n')
218 218 self.assertEqual(isp.indent_spaces, 0)
219 219
220 220 def test_dedent_pass(self):
221 221 isp = self.isp # shorthand
222 222 # should NOT cause dedent
223 223 isp.push('if 1:\n passes = 5')
224 224 self.assertEqual(isp.indent_spaces, 4)
225 225 isp.push('if 1:\n pass')
226 226 self.assertEqual(isp.indent_spaces, 0)
227 227 isp.push('if 1:\n pass ')
228 228 self.assertEqual(isp.indent_spaces, 0)
229 229
230 def test_dedent_break(self):
231 isp = self.isp # shorthand
232 # should NOT cause dedent
233 isp.push('while 1:\n breaks = 5')
234 self.assertEqual(isp.indent_spaces, 4)
235 isp.push('while 1:\n break')
236 self.assertEqual(isp.indent_spaces, 0)
237 isp.push('while 1:\n break ')
238 self.assertEqual(isp.indent_spaces, 0)
239
240 def test_dedent_continue(self):
241 isp = self.isp # shorthand
242 # should NOT cause dedent
243 isp.push('while 1:\n continues = 5')
244 self.assertEqual(isp.indent_spaces, 4)
245 isp.push('while 1:\n continue')
246 self.assertEqual(isp.indent_spaces, 0)
247 isp.push('while 1:\n continue ')
248 self.assertEqual(isp.indent_spaces, 0)
249
230 250 def test_dedent_raise(self):
231 251 isp = self.isp # shorthand
232 252 # should NOT cause dedent
233 253 isp.push('if 1:\n raised = 4')
234 254 self.assertEqual(isp.indent_spaces, 4)
235 255 isp.push('if 1:\n raise TypeError()')
236 256 self.assertEqual(isp.indent_spaces, 0)
237 257 isp.push('if 1:\n raise')
238 258 self.assertEqual(isp.indent_spaces, 0)
239 259 isp.push('if 1:\n raise ')
240 260 self.assertEqual(isp.indent_spaces, 0)
241 261
242 262 def test_dedent_return(self):
243 263 isp = self.isp # shorthand
244 264 # should NOT cause dedent
245 265 isp.push('if 1:\n returning = 4')
246 266 self.assertEqual(isp.indent_spaces, 4)
247 267 isp.push('if 1:\n return 5 + 493')
248 268 self.assertEqual(isp.indent_spaces, 0)
249 269 isp.push('if 1:\n return')
250 270 self.assertEqual(isp.indent_spaces, 0)
251 271 isp.push('if 1:\n return ')
252 272 self.assertEqual(isp.indent_spaces, 0)
253 273 isp.push('if 1:\n return(0)')
254 274 self.assertEqual(isp.indent_spaces, 0)
255 275
276 def test_dedent_yield(self):
277 isp = self.isp # shorthand
278 # should NOT cause dedent
279 isp.push('if 1:\n yielding = 4')
280 self.assertEqual(isp.indent_spaces, 4)
281 isp.push('if 1:\n yield 5 + 493')
282 self.assertEqual(isp.indent_spaces, 0)
283 isp.push('if 1:\n yield')
284 self.assertEqual(isp.indent_spaces, 0)
285 isp.push('if 1:\n yield ')
286 self.assertEqual(isp.indent_spaces, 0)
287 isp.push('if 1:\n yield(0)')
288 self.assertEqual(isp.indent_spaces, 0)
289
256 290 def test_push(self):
257 291 isp = self.isp
258 292 self.assertTrue(isp.push('x=1'))
259 293
260 294 def test_push2(self):
261 295 isp = self.isp
262 296 self.assertFalse(isp.push('if 1:'))
263 297 for line in [' x=1', '# a comment', ' y=2']:
264 298 self.assertTrue(isp.push(line))
265 299
266 300 def test_push3(self):
267 301 isp = self.isp
268 302 isp.push('if True:')
269 303 isp.push(' a = 1')
270 304 self.assertFalse(isp.push('b = [1,'))
271 305
272 306 def test_replace_mode(self):
273 307 isp = self.isp
274 308 isp.input_mode = 'cell'
275 309 isp.push('x=1')
276 310 self.assertEqual(isp.source, 'x=1\n')
277 311 isp.push('x=2')
278 312 self.assertEqual(isp.source, 'x=2\n')
279 313
280 314 def test_push_accepts_more(self):
281 315 isp = self.isp
282 316 isp.push('x=1')
283 317 self.assertFalse(isp.push_accepts_more())
284 318
285 319 def test_push_accepts_more2(self):
286 320 # In cell mode, inputs must be fed in whole blocks, so skip this test
287 321 if self.isp.input_mode == 'cell': return
288 322
289 323 isp = self.isp
290 324 isp.push('if 1:')
291 325 self.assertTrue(isp.push_accepts_more())
292 326 isp.push(' x=1')
293 327 self.assertTrue(isp.push_accepts_more())
294 328 isp.push('')
295 329 self.assertFalse(isp.push_accepts_more())
296 330
297 331 def test_push_accepts_more3(self):
298 332 isp = self.isp
299 333 isp.push("x = (2+\n3)")
300 334 self.assertFalse(isp.push_accepts_more())
301 335
302 336 def test_push_accepts_more4(self):
303 337 # In cell mode, inputs must be fed in whole blocks, so skip this test
304 338 if self.isp.input_mode == 'cell': return
305 339
306 340 isp = self.isp
307 341 # When a multiline statement contains parens or multiline strings, we
308 342 # shouldn't get confused.
309 343 # FIXME: we should be able to better handle de-dents in statements like
310 344 # multiline strings and multiline expressions (continued with \ or
311 345 # parens). Right now we aren't handling the indentation tracking quite
312 346 # correctly with this, though in practice it may not be too much of a
313 347 # problem. We'll need to see.
314 348 isp.push("if 1:")
315 349 isp.push(" x = (2+")
316 350 isp.push(" 3)")
317 351 self.assertTrue(isp.push_accepts_more())
318 352 isp.push(" y = 3")
319 353 self.assertTrue(isp.push_accepts_more())
320 354 isp.push('')
321 355 self.assertFalse(isp.push_accepts_more())
322 356
323 357 def test_push_accepts_more5(self):
324 358 # In cell mode, inputs must be fed in whole blocks, so skip this test
325 359 if self.isp.input_mode == 'cell': return
326 360
327 361 isp = self.isp
328 362 isp.push('try:')
329 363 isp.push(' a = 5')
330 364 isp.push('except:')
331 365 isp.push(' raise')
332 366 self.assertTrue(isp.push_accepts_more())
333 367
334 368 def test_continuation(self):
335 369 isp = self.isp
336 370 isp.push("import os, \\")
337 371 self.assertTrue(isp.push_accepts_more())
338 372 isp.push("sys")
339 373 self.assertFalse(isp.push_accepts_more())
340 374
341 375 def test_syntax_error(self):
342 376 isp = self.isp
343 377 # Syntax errors immediately produce a 'ready' block, so the invalid
344 378 # Python can be sent to the kernel for evaluation with possible ipython
345 379 # special-syntax conversion.
346 380 isp.push('run foo')
347 381 self.assertFalse(isp.push_accepts_more())
348 382
349 383 def test_unicode(self):
350 384 self.isp.push(u"PΓ©rez")
351 385 self.isp.push(u'\xc3\xa9')
352 386 self.isp.push(u"u'\xc3\xa9'")
353 387
354 388 def test_line_continuation(self):
355 389 """ Test issue #2108."""
356 390 isp = self.isp
357 391 # A blank line after a line continuation should not accept more
358 392 isp.push("1 \\\n\n")
359 393 self.assertFalse(isp.push_accepts_more())
360 394 # Whitespace after a \ is a SyntaxError. The only way to test that
361 395 # here is to test that push doesn't accept more (as with
362 396 # test_syntax_error() above).
363 397 isp.push(r"1 \ ")
364 398 self.assertFalse(isp.push_accepts_more())
365 399 # Even if the line is continuable (c.f. the regular Python
366 400 # interpreter)
367 401 isp.push(r"(1 \ ")
368 402 self.assertFalse(isp.push_accepts_more())
369 403
370 404 class InteractiveLoopTestCase(unittest.TestCase):
371 405 """Tests for an interactive loop like a python shell.
372 406 """
373 407 def check_ns(self, lines, ns):
374 408 """Validate that the given input lines produce the resulting namespace.
375 409
376 410 Note: the input lines are given exactly as they would be typed in an
377 411 auto-indenting environment, as mini_interactive_loop above already does
378 412 auto-indenting and prepends spaces to the input.
379 413 """
380 414 src = mini_interactive_loop(pseudo_input(lines))
381 415 test_ns = {}
382 416 exec src in test_ns
383 417 # We can't check that the provided ns is identical to the test_ns,
384 418 # because Python fills test_ns with extra keys (copyright, etc). But
385 419 # we can check that the given dict is *contained* in test_ns
386 420 for k,v in ns.iteritems():
387 421 self.assertEqual(test_ns[k], v)
388 422
389 423 def test_simple(self):
390 424 self.check_ns(['x=1'], dict(x=1))
391 425
392 426 def test_simple2(self):
393 427 self.check_ns(['if 1:', 'x=2'], dict(x=2))
394 428
395 429 def test_xy(self):
396 430 self.check_ns(['x=1; y=2'], dict(x=1, y=2))
397 431
398 432 def test_abc(self):
399 433 self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
400 434
401 435 def test_multi(self):
402 436 self.check_ns(['x =(1+','1+','2)'], dict(x=4))
403 437
404 438
405 439 def test_LineInfo():
406 440 """Simple test for LineInfo construction and str()"""
407 441 linfo = isp.LineInfo(' %cd /home')
408 442 nt.assert_equals(str(linfo), 'LineInfo [ |%|cd|/home]')
409 443
410 444 # Transformer tests
411 445 def transform_checker(tests, func):
412 446 """Utility to loop over test inputs"""
413 447 for inp, tr in tests:
414 448 nt.assert_equals(func(inp), tr)
415 449
416 450 # Data for all the syntax tests in the form of lists of pairs of
417 451 # raw/transformed input. We store it here as a global dict so that we can use
418 452 # it both within single-function tests and also to validate the behavior of the
419 453 # larger objects
420 454
421 455 syntax = \
422 456 dict(assign_system =
423 457 [(i,py3compat.u_format(o)) for i,o in \
424 458 [(u'a =! ls', "a = get_ipython().getoutput({u}'ls')"),
425 459 (u'b = !ls', "b = get_ipython().getoutput({u}'ls')"),
426 460 ('x=1', 'x=1'), # normal input is unmodified
427 461 (' ',' '), # blank lines are kept intact
428 462 ]],
429 463
430 464 assign_magic =
431 465 [(i,py3compat.u_format(o)) for i,o in \
432 466 [(u'a =% who', "a = get_ipython().magic({u}'who')"),
433 467 (u'b = %who', "b = get_ipython().magic({u}'who')"),
434 468 ('x=1', 'x=1'), # normal input is unmodified
435 469 (' ',' '), # blank lines are kept intact
436 470 ]],
437 471
438 472 classic_prompt =
439 473 [('>>> x=1', 'x=1'),
440 474 ('x=1', 'x=1'), # normal input is unmodified
441 475 (' ', ' '), # blank lines are kept intact
442 476 ('... ', ''), # continuation prompts
443 477 ],
444 478
445 479 ipy_prompt =
446 480 [('In [1]: x=1', 'x=1'),
447 481 ('x=1', 'x=1'), # normal input is unmodified
448 482 (' ',' '), # blank lines are kept intact
449 483 (' ....: ', ''), # continuation prompts
450 484 ],
451 485
452 486 # Tests for the escape transformer to leave normal code alone
453 487 escaped_noesc =
454 488 [ (' ', ' '),
455 489 ('x=1', 'x=1'),
456 490 ],
457 491
458 492 # System calls
459 493 escaped_shell =
460 494 [(i,py3compat.u_format(o)) for i,o in \
461 495 [ (u'!ls', "get_ipython().system({u}'ls')"),
462 496 # Double-escape shell, this means to capture the output of the
463 497 # subprocess and return it
464 498 (u'!!ls', "get_ipython().getoutput({u}'ls')"),
465 499 ]],
466 500
467 501 # Help/object info
468 502 escaped_help =
469 503 [(i,py3compat.u_format(o)) for i,o in \
470 504 [ (u'?', 'get_ipython().show_usage()'),
471 505 (u'?x1', "get_ipython().magic({u}'pinfo x1')"),
472 506 (u'??x2', "get_ipython().magic({u}'pinfo2 x2')"),
473 507 (u'?a.*s', "get_ipython().magic({u}'psearch a.*s')"),
474 508 (u'?%hist1', "get_ipython().magic({u}'pinfo %hist1')"),
475 509 (u'?%%hist2', "get_ipython().magic({u}'pinfo %%hist2')"),
476 510 (u'?abc = qwe', "get_ipython().magic({u}'pinfo abc')"),
477 511 ]],
478 512
479 513 end_help =
480 514 [(i,py3compat.u_format(o)) for i,o in \
481 515 [ (u'x3?', "get_ipython().magic({u}'pinfo x3')"),
482 516 (u'x4??', "get_ipython().magic({u}'pinfo2 x4')"),
483 517 (u'%hist1?', "get_ipython().magic({u}'pinfo %hist1')"),
484 518 (u'%hist2??', "get_ipython().magic({u}'pinfo2 %hist2')"),
485 519 (u'%%hist3?', "get_ipython().magic({u}'pinfo %%hist3')"),
486 520 (u'%%hist4??', "get_ipython().magic({u}'pinfo2 %%hist4')"),
487 521 (u'f*?', "get_ipython().magic({u}'psearch f*')"),
488 522 (u'ax.*aspe*?', "get_ipython().magic({u}'psearch ax.*aspe*')"),
489 523 (u'a = abc?', "get_ipython().set_next_input({u}'a = abc');"
490 524 "get_ipython().magic({u}'pinfo abc')"),
491 525 (u'a = abc.qe??', "get_ipython().set_next_input({u}'a = abc.qe');"
492 526 "get_ipython().magic({u}'pinfo2 abc.qe')"),
493 527 (u'a = *.items?', "get_ipython().set_next_input({u}'a = *.items');"
494 528 "get_ipython().magic({u}'psearch *.items')"),
495 529 (u'plot(a?', "get_ipython().set_next_input({u}'plot(a');"
496 530 "get_ipython().magic({u}'pinfo a')"),
497 531 (u'a*2 #comment?', 'a*2 #comment?'),
498 532 ]],
499 533
500 534 # Explicit magic calls
501 535 escaped_magic =
502 536 [(i,py3compat.u_format(o)) for i,o in \
503 537 [ (u'%cd', "get_ipython().magic({u}'cd')"),
504 538 (u'%cd /home', "get_ipython().magic({u}'cd /home')"),
505 539 # Backslashes need to be escaped.
506 540 (u'%cd C:\\User', "get_ipython().magic({u}'cd C:\\\\User')"),
507 541 (u' %magic', " get_ipython().magic({u}'magic')"),
508 542 ]],
509 543
510 544 # Quoting with separate arguments
511 545 escaped_quote =
512 546 [ (',f', 'f("")'),
513 547 (',f x', 'f("x")'),
514 548 (' ,f y', ' f("y")'),
515 549 (',f a b', 'f("a", "b")'),
516 550 ],
517 551
518 552 # Quoting with single argument
519 553 escaped_quote2 =
520 554 [ (';f', 'f("")'),
521 555 (';f x', 'f("x")'),
522 556 (' ;f y', ' f("y")'),
523 557 (';f a b', 'f("a b")'),
524 558 ],
525 559
526 560 # Simply apply parens
527 561 escaped_paren =
528 562 [ ('/f', 'f()'),
529 563 ('/f x', 'f(x)'),
530 564 (' /f y', ' f(y)'),
531 565 ('/f a b', 'f(a, b)'),
532 566 ],
533 567
534 568 # Check that we transform prompts before other transforms
535 569 mixed =
536 570 [(i,py3compat.u_format(o)) for i,o in \
537 571 [ (u'In [1]: %lsmagic', "get_ipython().magic({u}'lsmagic')"),
538 572 (u'>>> %lsmagic', "get_ipython().magic({u}'lsmagic')"),
539 573 (u'In [2]: !ls', "get_ipython().system({u}'ls')"),
540 574 (u'In [3]: abs?', "get_ipython().magic({u}'pinfo abs')"),
541 575 (u'In [4]: b = %who', "b = get_ipython().magic({u}'who')"),
542 576 ]],
543 577 )
544 578
545 579 # multiline syntax examples. Each of these should be a list of lists, with
546 580 # each entry itself having pairs of raw/transformed input. The union (with
547 581 # '\n'.join() of the transformed inputs is what the splitter should produce
548 582 # when fed the raw lines one at a time via push.
549 583 syntax_ml = \
550 584 dict(classic_prompt =
551 585 [ [('>>> for i in range(10):','for i in range(10):'),
552 586 ('... print i',' print i'),
553 587 ('... ', ''),
554 588 ],
555 589 ],
556 590
557 591 ipy_prompt =
558 592 [ [('In [24]: for i in range(10):','for i in range(10):'),
559 593 (' ....: print i',' print i'),
560 594 (' ....: ', ''),
561 595 ],
562 596 ],
563 597
564 598 multiline_datastructure =
565 599 [ [('>>> a = [1,','a = [1,'),
566 600 ('... 2]','2]'),
567 601 ],
568 602 ],
569 603 )
570 604
571 605
572 606 def test_assign_system():
573 607 tt.check_pairs(isp.transform_assign_system, syntax['assign_system'])
574 608
575 609
576 610 def test_assign_magic():
577 611 tt.check_pairs(isp.transform_assign_magic, syntax['assign_magic'])
578 612
579 613
580 614 def test_classic_prompt():
581 615 transform_checker(syntax['classic_prompt'], isp.transform_classic_prompt)
582 616 for example in syntax_ml['classic_prompt']:
583 617 transform_checker(example, isp.transform_classic_prompt)
584 618
585 619
586 620 def test_ipy_prompt():
587 621 transform_checker(syntax['ipy_prompt'], isp.transform_ipy_prompt)
588 622 for example in syntax_ml['ipy_prompt']:
589 623 transform_checker(example, isp.transform_ipy_prompt)
590 624
591 625 def test_end_help():
592 626 tt.check_pairs(isp.transform_help_end, syntax['end_help'])
593 627
594 628 def test_escaped_noesc():
595 629 tt.check_pairs(isp.transform_escaped, syntax['escaped_noesc'])
596 630
597 631
598 632 def test_escaped_shell():
599 633 tt.check_pairs(isp.transform_escaped, syntax['escaped_shell'])
600 634
601 635
602 636 def test_escaped_help():
603 637 tt.check_pairs(isp.transform_escaped, syntax['escaped_help'])
604 638
605 639
606 640 def test_escaped_magic():
607 641 tt.check_pairs(isp.transform_escaped, syntax['escaped_magic'])
608 642
609 643
610 644 def test_escaped_quote():
611 645 tt.check_pairs(isp.transform_escaped, syntax['escaped_quote'])
612 646
613 647
614 648 def test_escaped_quote2():
615 649 tt.check_pairs(isp.transform_escaped, syntax['escaped_quote2'])
616 650
617 651
618 652 def test_escaped_paren():
619 653 tt.check_pairs(isp.transform_escaped, syntax['escaped_paren'])
620 654
621 655
622 656 class IPythonInputTestCase(InputSplitterTestCase):
623 657 """By just creating a new class whose .isp is a different instance, we
624 658 re-run the same test battery on the new input splitter.
625 659
626 660 In addition, this runs the tests over the syntax and syntax_ml dicts that
627 661 were tested by individual functions, as part of the OO interface.
628 662
629 663 It also makes some checks on the raw buffer storage.
630 664 """
631 665
632 666 def setUp(self):
633 667 self.isp = isp.IPythonInputSplitter(input_mode='line')
634 668
635 669 def test_syntax(self):
636 670 """Call all single-line syntax tests from the main object"""
637 671 isp = self.isp
638 672 for example in syntax.itervalues():
639 673 for raw, out_t in example:
640 674 if raw.startswith(' '):
641 675 continue
642 676
643 677 isp.push(raw+'\n')
644 678 out, out_raw = isp.source_raw_reset()
645 679 self.assertEqual(out.rstrip(), out_t,
646 680 tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
647 681 self.assertEqual(out_raw.rstrip(), raw.rstrip())
648 682
649 683 def test_syntax_multiline(self):
650 684 isp = self.isp
651 685 for example in syntax_ml.itervalues():
652 686 out_t_parts = []
653 687 raw_parts = []
654 688 for line_pairs in example:
655 689 for lraw, out_t_part in line_pairs:
656 690 isp.push(lraw)
657 691 out_t_parts.append(out_t_part)
658 692 raw_parts.append(lraw)
659 693
660 694 out, out_raw = isp.source_raw_reset()
661 695 out_t = '\n'.join(out_t_parts).rstrip()
662 696 raw = '\n'.join(raw_parts).rstrip()
663 697 self.assertEqual(out.rstrip(), out_t)
664 698 self.assertEqual(out_raw.rstrip(), raw)
665 699
666 700
667 701 class BlockIPythonInputTestCase(IPythonInputTestCase):
668 702
669 703 # Deactivate tests that don't make sense for the block mode
670 704 test_push3 = test_split = lambda s: None
671 705
672 706 def setUp(self):
673 707 self.isp = isp.IPythonInputSplitter(input_mode='cell')
674 708
675 709 def test_syntax_multiline(self):
676 710 isp = self.isp
677 711 for example in syntax_ml.itervalues():
678 712 raw_parts = []
679 713 out_t_parts = []
680 714 for line_pairs in example:
681 715 for raw, out_t_part in line_pairs:
682 716 raw_parts.append(raw)
683 717 out_t_parts.append(out_t_part)
684 718
685 719 raw = '\n'.join(raw_parts)
686 720 out_t = '\n'.join(out_t_parts)
687 721
688 722 isp.push(raw)
689 723 out, out_raw = isp.source_raw_reset()
690 724 # Match ignoring trailing whitespace
691 725 self.assertEqual(out.rstrip(), out_t.rstrip())
692 726 self.assertEqual(out_raw.rstrip(), raw.rstrip())
693 727
694 728 def test_syntax_multiline_cell(self):
695 729 isp = self.isp
696 730 for example in syntax_ml.itervalues():
697 731
698 732 out_t_parts = []
699 733 for line_pairs in example:
700 734 raw = '\n'.join(r for r, _ in line_pairs)
701 735 out_t = '\n'.join(t for _,t in line_pairs)
702 736 out = isp.transform_cell(raw)
703 737 # Match ignoring trailing whitespace
704 738 self.assertEqual(out.rstrip(), out_t.rstrip())
705 739
706 740 #-----------------------------------------------------------------------------
707 741 # Main - use as a script, mostly for developer experiments
708 742 #-----------------------------------------------------------------------------
709 743
710 744 if __name__ == '__main__':
711 745 # A simple demo for interactive experimentation. This code will not get
712 746 # picked up by any test suite.
713 747 from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
714 748
715 749 # configure here the syntax to use, prompt and whether to autoindent
716 750 #isp, start_prompt = InputSplitter(), '>>> '
717 751 isp, start_prompt = IPythonInputSplitter(), 'In> '
718 752
719 753 autoindent = True
720 754 #autoindent = False
721 755
722 756 try:
723 757 while True:
724 758 prompt = start_prompt
725 759 while isp.push_accepts_more():
726 760 indent = ' '*isp.indent_spaces
727 761 if autoindent:
728 762 line = indent + raw_input(prompt+indent)
729 763 else:
730 764 line = raw_input(prompt)
731 765 isp.push(line)
732 766 prompt = '... '
733 767
734 768 # Here we just return input so we can use it in a test suite, but a
735 769 # real interpreter would instead send it for execution somewhere.
736 770 #src = isp.source; raise EOFError # dbg
737 771 src, raw = isp.source_raw_reset()
738 772 print 'Input source was:\n', src
739 773 print 'Raw source was:\n', raw
740 774 except EOFError:
741 775 print 'Bye'
742 776
743 777 # Tests for cell magics support
744 778
745 779 def test_last_blank():
746 780 nt.assert_false(isp.last_blank(''))
747 781 nt.assert_false(isp.last_blank('abc'))
748 782 nt.assert_false(isp.last_blank('abc\n'))
749 783 nt.assert_false(isp.last_blank('abc\na'))
750 784
751 785 nt.assert_true(isp.last_blank('\n'))
752 786 nt.assert_true(isp.last_blank('\n '))
753 787 nt.assert_true(isp.last_blank('abc\n '))
754 788 nt.assert_true(isp.last_blank('abc\n\n'))
755 789 nt.assert_true(isp.last_blank('abc\nd\n\n'))
756 790 nt.assert_true(isp.last_blank('abc\nd\ne\n\n'))
757 791 nt.assert_true(isp.last_blank('abc \n \n \n\n'))
758 792
759 793
760 794 def test_last_two_blanks():
761 795 nt.assert_false(isp.last_two_blanks(''))
762 796 nt.assert_false(isp.last_two_blanks('abc'))
763 797 nt.assert_false(isp.last_two_blanks('abc\n'))
764 798 nt.assert_false(isp.last_two_blanks('abc\n\na'))
765 799 nt.assert_false(isp.last_two_blanks('abc\n \n'))
766 800 nt.assert_false(isp.last_two_blanks('abc\n\n'))
767 801
768 802 nt.assert_true(isp.last_two_blanks('\n\n'))
769 803 nt.assert_true(isp.last_two_blanks('\n\n '))
770 804 nt.assert_true(isp.last_two_blanks('\n \n'))
771 805 nt.assert_true(isp.last_two_blanks('abc\n\n '))
772 806 nt.assert_true(isp.last_two_blanks('abc\n\n\n'))
773 807 nt.assert_true(isp.last_two_blanks('abc\n\n \n'))
774 808 nt.assert_true(isp.last_two_blanks('abc\n\n \n '))
775 809 nt.assert_true(isp.last_two_blanks('abc\n\n \n \n'))
776 810 nt.assert_true(isp.last_two_blanks('abc\nd\n\n\n'))
777 811 nt.assert_true(isp.last_two_blanks('abc\nd\ne\nf\n\n\n'))
778 812
779 813
780 814 class CellMagicsCommon(object):
781 815
782 816 def test_whole_cell(self):
783 817 src = "%%cellm line\nbody\n"
784 818 sp = self.sp
785 819 sp.push(src)
786 820 nt.assert_equal(sp.cell_magic_parts, ['body\n'])
787 821 out = sp.source
788 822 ref = u"get_ipython()._run_cached_cell_magic({u}'cellm', {u}'line')\n"
789 823 nt.assert_equal(out, py3compat.u_format(ref))
790 824
791 825 def tearDown(self):
792 826 self.sp.reset()
793 827
794 828
795 829 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
796 830 sp = isp.IPythonInputSplitter(input_mode='cell')
797 831
798 832 def test_incremental(self):
799 833 sp = self.sp
800 834 src = '%%cellm line2\n'
801 835 sp.push(src)
802 836 nt.assert_true(sp.push_accepts_more()) #1
803 837 src += '\n'
804 838 sp.push(src)
805 839 # Note: if we ever change the logic to allow full blank lines (see
806 840 # _handle_cell_magic), then the following test should change to true
807 841 nt.assert_false(sp.push_accepts_more()) #2
808 842 # By now, even with full blanks allowed, a second blank should signal
809 843 # the end. For now this test is only a redundancy safety, but don't
810 844 # delete it in case we change our mind and the previous one goes to
811 845 # true.
812 846 src += '\n'
813 847 sp.push(src)
814 848 nt.assert_false(sp.push_accepts_more()) #3
815 849
816 850
817 851 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
818 852 sp = isp.IPythonInputSplitter(input_mode='line')
819 853
820 854 def test_incremental(self):
821 855 sp = self.sp
822 856 sp.push('%%cellm line2\n')
823 857 nt.assert_true(sp.push_accepts_more()) #1
824 858 sp.push('\n')
825 859 nt.assert_false(sp.push_accepts_more()) #2
General Comments 0
You need to be logged in to leave comments. Login now