##// END OF EJS Templates
Fix RST misformatting.
Jason Grout -
Show More
@@ -1,914 +1,914 b''
1 1 """Analysis of text input into executable blocks.
2 2
3 3 The main class in this module, :class:`InputSplitter`, is designed to break
4 4 input from either interactive, line-by-line environments or block-based ones,
5 5 into standalone blocks that can be executed by Python as 'single' statements
6 6 (thus triggering sys.displayhook).
7 7
8 8 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
9 9 with full support for the extended IPython syntax (magics, system calls, etc).
10 10
11 11 For more details, see the class docstring below.
12 12
13 13 Syntax Transformations
14 14 ----------------------
15 15
16 16 One of the main jobs of the code in this file is to apply all syntax
17 17 transformations that make up 'the IPython language', i.e. magics, shell
18 18 escapes, etc. All transformations should be implemented as *fully stateless*
19 19 entities, that simply take one line as their input and return a line.
20 20 Internally for implementation purposes they may be a normal function or a
21 21 callable object, but the only input they receive will be a single line and they
22 22 should only return a line, without holding any data-dependent state between
23 23 calls.
24 24
25 25 As an example, the EscapedTransformer is a class so we can more clearly group
26 26 together the functionality of dispatching to individual functions based on the
27 27 starting escape character, but the only method for public use is its call
28 28 method.
29 29
30 30
31 31 ToDo
32 32 ----
33 33
34 34 - Should we make push() actually raise an exception once push_accepts_more()
35 35 returns False?
36 36
37 37 - Naming cleanups. The tr_* names aren't the most elegant, though now they are
38 38 at least just attributes of a class so not really very exposed.
39 39
40 40 - Think about the best way to support dynamic things: automagic, autocall,
41 41 macros, etc.
42 42
43 43 - Think of a better heuristic for the application of the transforms in
44 44 IPythonInputSplitter.push() than looking at the buffer ending in ':'. Idea:
45 45 track indentation change events (indent, dedent, nothing) and apply them only
46 46 if the indentation went up, but not otherwise.
47 47
48 48 - Think of the cleanest way for supporting user-specified transformations (the
49 49 user prefilters we had before).
50 50
51 51 Authors
52 52 -------
53 53
54 54 * Fernando Perez
55 55 * Brian Granger
56 56 """
57 57 #-----------------------------------------------------------------------------
58 58 # Copyright (C) 2010 The IPython Development Team
59 59 #
60 60 # Distributed under the terms of the BSD License. The full license is in
61 61 # the file COPYING, distributed as part of this software.
62 62 #-----------------------------------------------------------------------------
63 63
64 64 #-----------------------------------------------------------------------------
65 65 # Imports
66 66 #-----------------------------------------------------------------------------
67 67 # stdlib
68 68 import ast
69 69 import codeop
70 70 import re
71 71 import sys
72 72 import tokenize
73 73 from StringIO import StringIO
74 74
75 75 # IPython modules
76 76 from IPython.core.splitinput import split_user_input, LineInfo
77 77 from IPython.utils.py3compat import cast_unicode
78 78
79 79 #-----------------------------------------------------------------------------
80 80 # Globals
81 81 #-----------------------------------------------------------------------------
82 82
83 83 # The escape sequences that define the syntax transformations IPython will
84 84 # apply to user input. These can NOT be just changed here: many regular
85 85 # expressions and other parts of the code may use their hardcoded values, and
86 86 # for all intents and purposes they constitute the 'IPython syntax', so they
87 87 # should be considered fixed.
88 88
89 89 ESC_SHELL = '!' # Send line to underlying system shell
90 90 ESC_SH_CAP = '!!' # Send line to system shell and capture output
91 91 ESC_HELP = '?' # Find information about object
92 92 ESC_HELP2 = '??' # Find extra-detailed information about object
93 93 ESC_MAGIC = '%' # Call magic function
94 94 ESC_MAGIC2 = '%%' # Call cell-magic function
95 95 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
96 96 ESC_QUOTE2 = ';' # Quote all args as a single string, call
97 97 ESC_PAREN = '/' # Call first argument with rest of line as arguments
98 98
99 99 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
100 100 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
101 101 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
102 102
103 103 #-----------------------------------------------------------------------------
104 104 # Utilities
105 105 #-----------------------------------------------------------------------------
106 106
107 107 # FIXME: These are general-purpose utilities that later can be moved to the
108 108 # general ward. Kept here for now because we're being very strict about test
109 109 # coverage with this code, and this lets us ensure that we keep 100% coverage
110 110 # while developing.
111 111
112 112 # compiled regexps for autoindent management
113 113 dedent_re = re.compile('|'.join([
114 114 r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
115 115 r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
116 116 r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
117 117 r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
118 118 r'^\s+pass\s*$' # pass (optionally followed by trailing spaces)
119 119 ]))
120 120 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
121 121
122 122 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
123 123 # before pure comments
124 124 comment_line_re = re.compile('^\s*\#')
125 125
126 126
127 127 def num_ini_spaces(s):
128 128 """Return the number of initial spaces in a string.
129 129
130 130 Note that tabs are counted as a single space. For now, we do *not* support
131 131 mixing of tabs and spaces in the user's input.
132 132
133 133 Parameters
134 134 ----------
135 135 s : string
136 136
137 137 Returns
138 138 -------
139 139 n : int
140 140 """
141 141
142 142 ini_spaces = ini_spaces_re.match(s)
143 143 if ini_spaces:
144 144 return ini_spaces.end()
145 145 else:
146 146 return 0
147 147
148 148 def last_blank(src):
149 149 """Determine if the input source ends in a blank.
150 150
151 151 A blank is either a newline or a line consisting of whitespace.
152 152
153 153 Parameters
154 154 ----------
155 155 src : string
156 156 A single or multiline string.
157 157 """
158 158 if not src: return False
159 159 ll = src.splitlines()[-1]
160 160 return (ll == '') or ll.isspace()
161 161
162 162
163 163 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
164 164 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
165 165
166 166 def last_two_blanks(src):
167 167 """Determine if the input source ends in two blanks.
168 168
169 169 A blank is either a newline or a line consisting of whitespace.
170 170
171 171 Parameters
172 172 ----------
173 173 src : string
174 174 A single or multiline string.
175 175 """
176 176 if not src: return False
177 177 # The logic here is tricky: I couldn't get a regexp to work and pass all
178 178 # the tests, so I took a different approach: split the source by lines,
179 179 # grab the last two and prepend '###\n' as a stand-in for whatever was in
180 180 # the body before the last two lines. Then, with that structure, it's
181 181 # possible to analyze with two regexps. Not the most elegant solution, but
182 182 # it works. If anyone tries to change this logic, make sure to validate
183 183 # the whole test suite first!
184 184 new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
185 185 return (bool(last_two_blanks_re.match(new_src)) or
186 186 bool(last_two_blanks_re2.match(new_src)) )
187 187
188 188
189 189 def remove_comments(src):
190 190 """Remove all comments from input source.
191 191
192 192 Note: comments are NOT recognized inside of strings!
193 193
194 194 Parameters
195 195 ----------
196 196 src : string
197 197 A single or multiline input string.
198 198
199 199 Returns
200 200 -------
201 201 String with all Python comments removed.
202 202 """
203 203
204 204 return re.sub('#.*', '', src)
205 205
206 206 def has_comment(src):
207 207 """Indicate whether an input line has (i.e. ends in, or is) a comment.
208 208
209 209 This uses tokenize, so it can distinguish comments from # inside strings.
210 210
211 211 Parameters
212 212 ----------
213 213 src : string
214 214 A single line input string.
215 215
216 216 Returns
217 217 -------
218 218 Boolean: True if source has a comment.
219 219 """
220 220 readline = StringIO(src).readline
221 221 toktypes = set()
222 222 try:
223 223 for t in tokenize.generate_tokens(readline):
224 224 toktypes.add(t[0])
225 225 except tokenize.TokenError:
226 226 pass
227 227 return(tokenize.COMMENT in toktypes)
228 228
229 229
230 230 def get_input_encoding():
231 231 """Return the default standard input encoding.
232 232
233 233 If sys.stdin has no encoding, 'ascii' is returned."""
234 234 # There are strange environments for which sys.stdin.encoding is None. We
235 235 # ensure that a valid encoding is returned.
236 236 encoding = getattr(sys.stdin, 'encoding', None)
237 237 if encoding is None:
238 238 encoding = 'ascii'
239 239 return encoding
240 240
241 241 #-----------------------------------------------------------------------------
242 242 # Classes and functions for normal Python syntax handling
243 243 #-----------------------------------------------------------------------------
244 244
245 245 class InputSplitter(object):
246 246 """An object that can accumulate lines of Python source before execution.
247 247
248 248 This object is designed to be fed python source line-by-line, using
249 249 :meth:`push`. It will return on each push whether the currently pushed
250 250 code could be executed already. In addition, it provides a method called
251 251 :meth:`push_accepts_more` that can be used to query whether more input
252 252 can be pushed into a single interactive block.
253 253
254 254 This is a simple example of how an interactive terminal-based client can use
255 255 this tool::
256 256
257 257 isp = InputSplitter()
258 258 while isp.push_accepts_more():
259 259 indent = ' '*isp.indent_spaces
260 260 prompt = '>>> ' + indent
261 261 line = indent + raw_input(prompt)
262 262 isp.push(line)
263 263 print 'Input source was:\n', isp.source_reset(),
264 264 """
265 265 # Number of spaces of indentation computed from input that has been pushed
266 266 # so far. This is the attributes callers should query to get the current
267 267 # indentation level, in order to provide auto-indent facilities.
268 268 indent_spaces = 0
269 269 # String, indicating the default input encoding. It is computed by default
270 270 # at initialization time via get_input_encoding(), but it can be reset by a
271 271 # client with specific knowledge of the encoding.
272 272 encoding = ''
273 273 # String where the current full source input is stored, properly encoded.
274 274 # Reading this attribute is the normal way of querying the currently pushed
275 275 # source code, that has been properly encoded.
276 276 source = ''
277 277 # Code object corresponding to the current source. It is automatically
278 278 # synced to the source, so it can be queried at any time to obtain the code
279 279 # object; it will be None if the source doesn't compile to valid Python.
280 280 code = None
281 281 # Input mode
282 282 input_mode = 'line'
283 283
284 284 # Private attributes
285 285
286 286 # List with lines of input accumulated so far
287 287 _buffer = None
288 288 # Command compiler
289 289 _compile = None
290 290 # Mark when input has changed indentation all the way back to flush-left
291 291 _full_dedent = False
292 292 # Boolean indicating whether the current block is complete
293 293 _is_complete = None
294 294
295 295 def __init__(self, input_mode=None):
296 296 """Create a new InputSplitter instance.
297 297
298 298 Parameters
299 299 ----------
300 300 input_mode : str
301 301
302 302 One of ['line', 'cell']; default is 'line'.
303 303
304 304 The input_mode parameter controls how new inputs are used when fed via
305 305 the :meth:`push` method:
306 306
307 307 - 'line': meant for line-oriented clients, inputs are appended one at a
308 308 time to the internal buffer and the whole buffer is compiled.
309 309
310 310 - 'cell': meant for clients that can edit multi-line 'cells' of text at
311 311 a time. A cell can contain one or more blocks that can be compile in
312 312 'single' mode by Python. In this mode, each new input new input
313 313 completely replaces all prior inputs. Cell mode is thus equivalent
314 314 to prepending a full reset() to every push() call.
315 315 """
316 316 self._buffer = []
317 317 self._compile = codeop.CommandCompiler()
318 318 self.encoding = get_input_encoding()
319 319 self.input_mode = InputSplitter.input_mode if input_mode is None \
320 320 else input_mode
321 321
322 322 def reset(self):
323 323 """Reset the input buffer and associated state."""
324 324 self.indent_spaces = 0
325 325 self._buffer[:] = []
326 326 self.source = ''
327 327 self.code = None
328 328 self._is_complete = False
329 329 self._full_dedent = False
330 330
331 331 def source_reset(self):
332 332 """Return the input source and perform a full reset.
333 333 """
334 334 out = self.source
335 335 self.reset()
336 336 return out
337 337
338 338 def push(self, lines):
339 339 """Push one or more lines of input.
340 340
341 341 This stores the given lines and returns a status code indicating
342 342 whether the code forms a complete Python block or not.
343 343
344 344 Any exceptions generated in compilation are swallowed, but if an
345 345 exception was produced, the method returns True.
346 346
347 347 Parameters
348 348 ----------
349 349 lines : string
350 350 One or more lines of Python input.
351 351
352 352 Returns
353 353 -------
354 354 is_complete : boolean
355 355 True if the current input source (the result of the current input
356 plus prior inputs) forms a complete Python execution block. Note that
357 this value is also stored as a private attribute (_is_complete), so it
358 can be queried at any time.
356 plus prior inputs) forms a complete Python execution block. Note that
357 this value is also stored as a private attribute (``_is_complete``), so it
358 can be queried at any time.
359 359 """
360 360 if self.input_mode == 'cell':
361 361 self.reset()
362 362
363 363 self._store(lines)
364 364 source = self.source
365 365
366 366 # Before calling _compile(), reset the code object to None so that if an
367 367 # exception is raised in compilation, we don't mislead by having
368 368 # inconsistent code/source attributes.
369 369 self.code, self._is_complete = None, None
370 370
371 371 # Honor termination lines properly
372 372 if source.rstrip().endswith('\\'):
373 373 return False
374 374
375 375 self._update_indent(lines)
376 376 try:
377 377 self.code = self._compile(source, symbol="exec")
378 378 # Invalid syntax can produce any of a number of different errors from
379 379 # inside the compiler, so we have to catch them all. Syntax errors
380 380 # immediately produce a 'ready' block, so the invalid Python can be
381 381 # sent to the kernel for evaluation with possible ipython
382 382 # special-syntax conversion.
383 383 except (SyntaxError, OverflowError, ValueError, TypeError,
384 384 MemoryError):
385 385 self._is_complete = True
386 386 else:
387 387 # Compilation didn't produce any exceptions (though it may not have
388 388 # given a complete code object)
389 389 self._is_complete = self.code is not None
390 390
391 391 return self._is_complete
392 392
393 393 def push_accepts_more(self):
394 394 """Return whether a block of interactive input can accept more input.
395 395
396 396 This method is meant to be used by line-oriented frontends, who need to
397 397 guess whether a block is complete or not based solely on prior and
398 398 current input lines. The InputSplitter considers it has a complete
399 399 interactive block and will not accept more input only when either a
400 400 SyntaxError is raised, or *all* of the following are true:
401 401
402 402 1. The input compiles to a complete statement.
403 403
404 404 2. The indentation level is flush-left (because if we are indented,
405 405 like inside a function definition or for loop, we need to keep
406 406 reading new input).
407 407
408 408 3. There is one extra line consisting only of whitespace.
409 409
410 410 Because of condition #3, this method should be used only by
411 411 *line-oriented* frontends, since it means that intermediate blank lines
412 412 are not allowed in function definitions (or any other indented block).
413 413
414 414 If the current input produces a syntax error, this method immediately
415 415 returns False but does *not* raise the syntax error exception, as
416 416 typically clients will want to send invalid syntax to an execution
417 417 backend which might convert the invalid syntax into valid Python via
418 418 one of the dynamic IPython mechanisms.
419 419 """
420 420
421 421 # With incomplete input, unconditionally accept more
422 422 if not self._is_complete:
423 423 return True
424 424
425 425 # If we already have complete input and we're flush left, the answer
426 426 # depends. In line mode, if there hasn't been any indentation,
427 427 # that's it. If we've come back from some indentation, we need
428 428 # the blank final line to finish.
429 429 # In cell mode, we need to check how many blocks the input so far
430 430 # compiles into, because if there's already more than one full
431 431 # independent block of input, then the client has entered full
432 432 # 'cell' mode and is feeding lines that each is complete. In this
433 433 # case we should then keep accepting. The Qt terminal-like console
434 434 # does precisely this, to provide the convenience of terminal-like
435 435 # input of single expressions, but allowing the user (with a
436 436 # separate keystroke) to switch to 'cell' mode and type multiple
437 437 # expressions in one shot.
438 438 if self.indent_spaces==0:
439 439 if self.input_mode=='line':
440 440 if not self._full_dedent:
441 441 return False
442 442 else:
443 443 try:
444 444 code_ast = ast.parse(u''.join(self._buffer))
445 445 except Exception:
446 446 return False
447 447 else:
448 448 if len(code_ast.body) == 1:
449 449 return False
450 450
451 451 # When input is complete, then termination is marked by an extra blank
452 452 # line at the end.
453 453 last_line = self.source.splitlines()[-1]
454 454 return bool(last_line and not last_line.isspace())
455 455
456 456 #------------------------------------------------------------------------
457 457 # Private interface
458 458 #------------------------------------------------------------------------
459 459
460 460 def _find_indent(self, line):
461 461 """Compute the new indentation level for a single line.
462 462
463 463 Parameters
464 464 ----------
465 465 line : str
466 466 A single new line of non-whitespace, non-comment Python input.
467 467
468 468 Returns
469 469 -------
470 470 indent_spaces : int
471 471 New value for the indent level (it may be equal to self.indent_spaces
472 472 if indentation doesn't change.
473 473
474 474 full_dedent : boolean
475 475 Whether the new line causes a full flush-left dedent.
476 476 """
477 477 indent_spaces = self.indent_spaces
478 478 full_dedent = self._full_dedent
479 479
480 480 inisp = num_ini_spaces(line)
481 481 if inisp < indent_spaces:
482 482 indent_spaces = inisp
483 483 if indent_spaces <= 0:
484 484 #print 'Full dedent in text',self.source # dbg
485 485 full_dedent = True
486 486
487 487 if line.rstrip()[-1] == ':':
488 488 indent_spaces += 4
489 489 elif dedent_re.match(line):
490 490 indent_spaces -= 4
491 491 if indent_spaces <= 0:
492 492 full_dedent = True
493 493
494 494 # Safety
495 495 if indent_spaces < 0:
496 496 indent_spaces = 0
497 497 #print 'safety' # dbg
498 498
499 499 return indent_spaces, full_dedent
500 500
501 501 def _update_indent(self, lines):
502 502 for line in remove_comments(lines).splitlines():
503 503 if line and not line.isspace():
504 504 self.indent_spaces, self._full_dedent = self._find_indent(line)
505 505
506 506 def _store(self, lines, buffer=None, store='source'):
507 507 """Store one or more lines of input.
508 508
509 509 If input lines are not newline-terminated, a newline is automatically
510 510 appended."""
511 511
512 512 if buffer is None:
513 513 buffer = self._buffer
514 514
515 515 if lines.endswith('\n'):
516 516 buffer.append(lines)
517 517 else:
518 518 buffer.append(lines+'\n')
519 519 setattr(self, store, self._set_source(buffer))
520 520
521 521 def _set_source(self, buffer):
522 522 return u''.join(buffer)
523 523
524 524
525 525 #-----------------------------------------------------------------------------
526 526 # Functions and classes for IPython-specific syntactic support
527 527 #-----------------------------------------------------------------------------
528 528
529 529 # The escaped translators ALL receive a line where their own escape has been
530 530 # stripped. Only '?' is valid at the end of the line, all others can only be
531 531 # placed at the start.
532 532
533 533 # Transformations of the special syntaxes that don't rely on an explicit escape
534 534 # character but instead on patterns on the input line
535 535
536 536 # The core transformations are implemented as standalone functions that can be
537 537 # tested and validated in isolation. Each of these uses a regexp, we
538 538 # pre-compile these and keep them close to each function definition for clarity
539 539
540 540 _assign_system_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
541 541 r'\s*=\s*!\s*(?P<cmd>.*)')
542 542
543 543 def transform_assign_system(line):
544 544 """Handle the `files = !ls` syntax."""
545 545 m = _assign_system_re.match(line)
546 546 if m is not None:
547 547 cmd = m.group('cmd')
548 548 lhs = m.group('lhs')
549 549 new_line = '%s = get_ipython().getoutput(%r)' % (lhs, cmd)
550 550 return new_line
551 551 return line
552 552
553 553
554 554 _assign_magic_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
555 555 r'\s*=\s*%\s*(?P<cmd>.*)')
556 556
557 557 def transform_assign_magic(line):
558 558 """Handle the `a = %who` syntax."""
559 559 m = _assign_magic_re.match(line)
560 560 if m is not None:
561 561 cmd = m.group('cmd')
562 562 lhs = m.group('lhs')
563 563 new_line = '%s = get_ipython().magic(%r)' % (lhs, cmd)
564 564 return new_line
565 565 return line
566 566
567 567
568 568 _classic_prompt_re = re.compile(r'^([ \t]*>>> |^[ \t]*\.\.\. )')
569 569
570 570 def transform_classic_prompt(line):
571 571 """Handle inputs that start with '>>> ' syntax."""
572 572
573 573 if not line or line.isspace():
574 574 return line
575 575 m = _classic_prompt_re.match(line)
576 576 if m:
577 577 return line[len(m.group(0)):]
578 578 else:
579 579 return line
580 580
581 581
582 582 _ipy_prompt_re = re.compile(r'^([ \t]*In \[\d+\]: |^[ \t]*\ \ \ \.\.\.+: )')
583 583
584 584 def transform_ipy_prompt(line):
585 585 """Handle inputs that start classic IPython prompt syntax."""
586 586
587 587 if not line or line.isspace():
588 588 return line
589 589 #print 'LINE: %r' % line # dbg
590 590 m = _ipy_prompt_re.match(line)
591 591 if m:
592 592 #print 'MATCH! %r -> %r' % (line, line[len(m.group(0)):]) # dbg
593 593 return line[len(m.group(0)):]
594 594 else:
595 595 return line
596 596
597 597
598 598 def _make_help_call(target, esc, lspace, next_input=None):
599 599 """Prepares a pinfo(2)/psearch call from a target name and the escape
600 600 (i.e. ? or ??)"""
601 601 method = 'pinfo2' if esc == '??' \
602 602 else 'psearch' if '*' in target \
603 603 else 'pinfo'
604 604 arg = " ".join([method, target])
605 605 if next_input is None:
606 606 return '%sget_ipython().magic(%r)' % (lspace, arg)
607 607 else:
608 608 return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
609 609 (lspace, next_input, arg)
610 610
611 611
612 612 _initial_space_re = re.compile(r'\s*')
613 613
614 614 _help_end_re = re.compile(r"""(%{0,2}
615 615 [a-zA-Z_*][\w*]* # Variable name
616 616 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
617 617 )
618 618 (\?\??)$ # ? or ??""",
619 619 re.VERBOSE)
620 620
621 621
622 622 def transform_help_end(line):
623 623 """Translate lines with ?/?? at the end"""
624 624 m = _help_end_re.search(line)
625 625 if m is None or has_comment(line):
626 626 return line
627 627 target = m.group(1)
628 628 esc = m.group(3)
629 629 lspace = _initial_space_re.match(line).group(0)
630 630
631 631 # If we're mid-command, put it back on the next prompt for the user.
632 632 next_input = line.rstrip('?') if line.strip() != m.group(0) else None
633 633
634 634 return _make_help_call(target, esc, lspace, next_input)
635 635
636 636
637 637 class EscapedTransformer(object):
638 638 """Class to transform lines that are explicitly escaped out."""
639 639
640 640 def __init__(self):
641 641 tr = { ESC_SHELL : self._tr_system,
642 642 ESC_SH_CAP : self._tr_system2,
643 643 ESC_HELP : self._tr_help,
644 644 ESC_HELP2 : self._tr_help,
645 645 ESC_MAGIC : self._tr_magic,
646 646 ESC_QUOTE : self._tr_quote,
647 647 ESC_QUOTE2 : self._tr_quote2,
648 648 ESC_PAREN : self._tr_paren }
649 649 self.tr = tr
650 650
651 651 # Support for syntax transformations that use explicit escapes typed by the
652 652 # user at the beginning of a line
653 653 @staticmethod
654 654 def _tr_system(line_info):
655 655 "Translate lines escaped with: !"
656 656 cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
657 657 return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
658 658
659 659 @staticmethod
660 660 def _tr_system2(line_info):
661 661 "Translate lines escaped with: !!"
662 662 cmd = line_info.line.lstrip()[2:]
663 663 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
664 664
665 665 @staticmethod
666 666 def _tr_help(line_info):
667 667 "Translate lines escaped with: ?/??"
668 668 # A naked help line should just fire the intro help screen
669 669 if not line_info.line[1:]:
670 670 return 'get_ipython().show_usage()'
671 671
672 672 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
673 673
674 674 @staticmethod
675 675 def _tr_magic(line_info):
676 676 "Translate lines escaped with: %"
677 677 tpl = '%sget_ipython().magic(%r)'
678 678 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
679 679 return tpl % (line_info.pre, cmd)
680 680
681 681 @staticmethod
682 682 def _tr_quote(line_info):
683 683 "Translate lines escaped with: ,"
684 684 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
685 685 '", "'.join(line_info.the_rest.split()) )
686 686
687 687 @staticmethod
688 688 def _tr_quote2(line_info):
689 689 "Translate lines escaped with: ;"
690 690 return '%s%s("%s")' % (line_info.pre, line_info.ifun,
691 691 line_info.the_rest)
692 692
693 693 @staticmethod
694 694 def _tr_paren(line_info):
695 695 "Translate lines escaped with: /"
696 696 return '%s%s(%s)' % (line_info.pre, line_info.ifun,
697 697 ", ".join(line_info.the_rest.split()))
698 698
699 699 def __call__(self, line):
700 700 """Class to transform lines that are explicitly escaped out.
701 701
702 702 This calls the above _tr_* static methods for the actual line
703 703 translations."""
704 704
705 705 # Empty lines just get returned unmodified
706 706 if not line or line.isspace():
707 707 return line
708 708
709 709 # Get line endpoints, where the escapes can be
710 710 line_info = LineInfo(line)
711 711
712 712 if not line_info.esc in self.tr:
713 713 # If we don't recognize the escape, don't modify the line
714 714 return line
715 715
716 716 return self.tr[line_info.esc](line_info)
717 717
718 718
719 719 # A function-looking object to be used by the rest of the code. The purpose of
720 720 # the class in this case is to organize related functionality, more than to
721 721 # manage state.
722 722 transform_escaped = EscapedTransformer()
723 723
724 724
725 725 class IPythonInputSplitter(InputSplitter):
726 726 """An input splitter that recognizes all of IPython's special syntax."""
727 727
728 728 # String with raw, untransformed input.
729 729 source_raw = ''
730 730
731 731 # Flag to track when we're in the middle of processing a cell magic, since
732 732 # the logic has to change. In that case, we apply no transformations at
733 733 # all.
734 734 processing_cell_magic = False
735 735
736 736 # Storage for all blocks of input that make up a cell magic
737 737 cell_magic_parts = []
738 738
739 739 # Private attributes
740 740
741 741 # List with lines of raw input accumulated so far.
742 742 _buffer_raw = None
743 743
744 744 def __init__(self, input_mode=None):
745 745 super(IPythonInputSplitter, self).__init__(input_mode)
746 746 self._buffer_raw = []
747 747 self._validate = True
748 748
749 749 def reset(self):
750 750 """Reset the input buffer and associated state."""
751 751 super(IPythonInputSplitter, self).reset()
752 752 self._buffer_raw[:] = []
753 753 self.source_raw = ''
754 754 self.cell_magic_parts = []
755 755 self.processing_cell_magic = False
756 756
757 757 def source_raw_reset(self):
758 758 """Return input and raw source and perform a full reset.
759 759 """
760 760 out = self.source
761 761 out_r = self.source_raw
762 762 self.reset()
763 763 return out, out_r
764 764
765 765 def push_accepts_more(self):
766 766 if self.processing_cell_magic:
767 767 return not self._is_complete
768 768 else:
769 769 return super(IPythonInputSplitter, self).push_accepts_more()
770 770
771 771 def _handle_cell_magic(self, lines):
772 772 """Process lines when they start with %%, which marks cell magics.
773 773 """
774 774 self.processing_cell_magic = True
775 775 first, _, body = lines.partition('\n')
776 776 magic_name, _, line = first.partition(' ')
777 777 magic_name = magic_name.lstrip(ESC_MAGIC)
778 778 # We store the body of the cell and create a call to a method that
779 779 # will use this stored value. This is ugly, but it's a first cut to
780 780 # get it all working, as right now changing the return API of our
781 781 # methods would require major refactoring.
782 782 self.cell_magic_parts = [body]
783 783 tpl = 'get_ipython()._run_cached_cell_magic(%r, %r)'
784 784 tlines = tpl % (magic_name, line)
785 785 self._store(tlines)
786 786 self._store(lines, self._buffer_raw, 'source_raw')
787 787 # We can actually choose whether to allow for single blank lines here
788 788 # during input for clients that use cell mode to decide when to stop
789 789 # pushing input (currently only the Qt console).
790 790 # My first implementation did that, and then I realized it wasn't
791 791 # consistent with the terminal behavior, so I've reverted it to one
792 792 # line. But I'm leaving it here so we can easily test both behaviors,
793 793 # I kind of liked having full blank lines allowed in the cell magics...
794 794 #self._is_complete = last_two_blanks(lines)
795 795 self._is_complete = last_blank(lines)
796 796 return self._is_complete
797 797
798 798 def _line_mode_cell_append(self, lines):
799 799 """Append new content for a cell magic in line mode.
800 800 """
801 801 # Only store the raw input. Lines beyond the first one are only only
802 802 # stored for history purposes; for execution the caller will grab the
803 803 # magic pieces from cell_magic_parts and will assemble the cell body
804 804 self._store(lines, self._buffer_raw, 'source_raw')
805 805 self.cell_magic_parts.append(lines)
806 806 # Find out if the last stored block has a whitespace line as its
807 807 # last line and also this line is whitespace, case in which we're
808 808 # done (two contiguous blank lines signal termination). Note that
809 809 # the storage logic *enforces* that every stored block is
810 810 # newline-terminated, so we grab everything but the last character
811 811 # so we can have the body of the block alone.
812 812 last_block = self.cell_magic_parts[-1]
813 813 self._is_complete = last_blank(last_block) and lines.isspace()
814 814 return self._is_complete
815 815
816 816 def transform_cell(self, cell):
817 817 """Process and translate a cell of input.
818 818 """
819 819 self.reset()
820 820 self.push(cell)
821 821 return self.source_reset()
822 822
823 823 def push(self, lines):
824 824 """Push one or more lines of IPython input.
825 825
826 826 This stores the given lines and returns a status code indicating
827 827 whether the code forms a complete Python block or not, after processing
828 828 all input lines for special IPython syntax.
829 829
830 830 Any exceptions generated in compilation are swallowed, but if an
831 831 exception was produced, the method returns True.
832 832
833 833 Parameters
834 834 ----------
835 835 lines : string
836 836 One or more lines of Python input.
837 837
838 838 Returns
839 839 -------
840 840 is_complete : boolean
841 841 True if the current input source (the result of the current input
842 842 plus prior inputs) forms a complete Python execution block. Note that
843 843 this value is also stored as a private attribute (_is_complete), so it
844 844 can be queried at any time.
845 845 """
846 846 if not lines:
847 847 return super(IPythonInputSplitter, self).push(lines)
848 848
849 849 # We must ensure all input is pure unicode
850 850 lines = cast_unicode(lines, self.encoding)
851 851
852 852 # If the entire input block is a cell magic, return after handling it
853 853 # as the rest of the transformation logic should be skipped.
854 854 if lines.startswith('%%') and not \
855 855 (len(lines.splitlines()) == 1 and lines.strip().endswith('?')):
856 856 return self._handle_cell_magic(lines)
857 857
858 858 # In line mode, a cell magic can arrive in separate pieces
859 859 if self.input_mode == 'line' and self.processing_cell_magic:
860 860 return self._line_mode_cell_append(lines)
861 861
862 862 # The rest of the processing is for 'normal' content, i.e. IPython
863 863 # source that we process through our transformations pipeline.
864 864 lines_list = lines.splitlines()
865 865
866 866 transforms = [transform_ipy_prompt, transform_classic_prompt,
867 867 transform_help_end, transform_escaped,
868 868 transform_assign_system, transform_assign_magic]
869 869
870 870 # Transform logic
871 871 #
872 872 # We only apply the line transformers to the input if we have either no
873 873 # input yet, or complete input, or if the last line of the buffer ends
874 874 # with ':' (opening an indented block). This prevents the accidental
875 875 # transformation of escapes inside multiline expressions like
876 876 # triple-quoted strings or parenthesized expressions.
877 877 #
878 878 # The last heuristic, while ugly, ensures that the first line of an
879 879 # indented block is correctly transformed.
880 880 #
881 881 # FIXME: try to find a cleaner approach for this last bit.
882 882
883 883 # If we were in 'block' mode, since we're going to pump the parent
884 884 # class by hand line by line, we need to temporarily switch out to
885 885 # 'line' mode, do a single manual reset and then feed the lines one
886 886 # by one. Note that this only matters if the input has more than one
887 887 # line.
888 888 changed_input_mode = False
889 889
890 890 if self.input_mode == 'cell':
891 891 self.reset()
892 892 changed_input_mode = True
893 893 saved_input_mode = 'cell'
894 894 self.input_mode = 'line'
895 895
896 896 # Store raw source before applying any transformations to it. Note
897 897 # that this must be done *after* the reset() call that would otherwise
898 898 # flush the buffer.
899 899 self._store(lines, self._buffer_raw, 'source_raw')
900 900
901 901 try:
902 902 push = super(IPythonInputSplitter, self).push
903 903 buf = self._buffer
904 904 for line in lines_list:
905 905 if self._is_complete or not buf or \
906 906 (buf and buf[-1].rstrip().endswith((':', ','))):
907 907 for f in transforms:
908 908 line = f(line)
909 909
910 910 out = push(line)
911 911 finally:
912 912 if changed_input_mode:
913 913 self.input_mode = saved_input_mode
914 914 return out
General Comments 0
You need to be logged in to leave comments. Login now