##// END OF EJS Templates
Merge pull request #12884 from MrMino/vi_mode_prompt_strip...
Matthias Bussonnier -
r26497:9cbb3ba6 merge
parent child Browse files
Show More
@@ -0,0 +1,29 b''
1 Automatic Vi prompt stripping
2 =============================
3
4 When pasting code into IPython, it will strip the leading prompt characters if
5 there are any. For example, you can paste the following code into the console -
6 it will still work, even though each line is prefixed with prompts (`In`,
7 `Out`)::
8
9 In [1]: 2 * 2 == 4
10 Out[1]: True
11
12 In [2]: print("This still works as pasted")
13
14
15 Previously, this was not the case for the Vi-mode prompts::
16
17 In [1]: [ins] In [13]: 2 * 2 == 4
18 ...: Out[13]: True
19 ...:
20 File "<ipython-input-1-727bb88eaf33>", line 1
21 [ins] In [13]: 2 * 2 == 4
22 ^
23 SyntaxError: invalid syntax
24
25 This is now fixed, and Vi prompt prefixes - ``[ins]`` and ``[nav]`` - are
26 skipped just as the normal ``In`` would be.
27
28 IPython shell can be started in the Vi mode using ``ipython
29 --TerminalInteractiveShell.editing_mode=vi``
@@ -1,752 +1,775 b''
1 1 """Input transformer machinery to support IPython special syntax.
2 2
3 3 This includes the machinery to recognise and transform ``%magic`` commands,
4 4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 5
6 6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 7 deprecated in 7.0.
8 8 """
9 9
10 10 # Copyright (c) IPython Development Team.
11 11 # Distributed under the terms of the Modified BSD License.
12 12
13 13 from codeop import compile_command
14 14 import re
15 15 import tokenize
16 16 from typing import List, Tuple, Optional, Any
17 17 import warnings
18 18
19 19 _indent_re = re.compile(r'^[ \t]+')
20 20
21 21 def leading_empty_lines(lines):
22 22 """Remove leading empty lines
23 23
24 24 If the leading lines are empty or contain only whitespace, they will be
25 25 removed.
26 26 """
27 27 if not lines:
28 28 return lines
29 29 for i, line in enumerate(lines):
30 30 if line and not line.isspace():
31 31 return lines[i:]
32 32 return lines
33 33
34 34 def leading_indent(lines):
35 35 """Remove leading indentation.
36 36
37 37 If the first line starts with a spaces or tabs, the same whitespace will be
38 38 removed from each following line in the cell.
39 39 """
40 40 if not lines:
41 41 return lines
42 42 m = _indent_re.match(lines[0])
43 43 if not m:
44 44 return lines
45 45 space = m.group(0)
46 46 n = len(space)
47 47 return [l[n:] if l.startswith(space) else l
48 48 for l in lines]
49 49
50 50 class PromptStripper:
51 51 """Remove matching input prompts from a block of input.
52 52
53 53 Parameters
54 54 ----------
55 55 prompt_re : regular expression
56 56 A regular expression matching any input prompt (including continuation,
57 57 e.g. ``...``)
58 58 initial_re : regular expression, optional
59 59 A regular expression matching only the initial prompt, but not continuation.
60 60 If no initial expression is given, prompt_re will be used everywhere.
61 61 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
62 62 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
63 63
64 64 Notes
65 65 -----
66 66
67 67 If initial_re and prompt_re differ,
68 68 only initial_re will be tested against the first line.
69 69 If any prompt is found on the first two lines,
70 70 prompts will be stripped from the rest of the block.
71 71 """
72 72 def __init__(self, prompt_re, initial_re=None):
73 73 self.prompt_re = prompt_re
74 74 self.initial_re = initial_re or prompt_re
75 75
76 76 def _strip(self, lines):
77 77 return [self.prompt_re.sub('', l, count=1) for l in lines]
78 78
79 79 def __call__(self, lines):
80 80 if not lines:
81 81 return lines
82 82 if self.initial_re.match(lines[0]) or \
83 83 (len(lines) > 1 and self.prompt_re.match(lines[1])):
84 84 return self._strip(lines)
85 85 return lines
86 86
87 87 classic_prompt = PromptStripper(
88 88 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
89 89 initial_re=re.compile(r'^>>>( |$)')
90 90 )
91 91
92 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
92 ipython_prompt = PromptStripper(
93 re.compile(
94 r"""
95 ^( # Match from the beginning of a line, either:
96
97 # 1. First-line prompt:
98 ((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there
99 In\ # The 'In' of the prompt, with a space
100 \[\d+\]: # Command index, as displayed in the prompt
101 \ # With a mandatory trailing space
102
103 | # ... or ...
104
105 # 2. The three dots of the multiline prompt
106 \s* # All leading whitespace characters
107 \.{3,}: # The three (or more) dots
108 \ ? # With an optional trailing space
109
110 )
111 """,
112 re.VERBOSE,
113 )
114 )
115
93 116
94 117 def cell_magic(lines):
95 118 if not lines or not lines[0].startswith('%%'):
96 119 return lines
97 120 if re.match(r'%%\w+\?', lines[0]):
98 121 # This case will be handled by help_end
99 122 return lines
100 123 magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
101 124 body = ''.join(lines[1:])
102 125 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
103 126 % (magic_name, first_line, body)]
104 127
105 128
106 129 def _find_assign_op(token_line) -> Optional[int]:
107 130 """Get the index of the first assignment in the line ('=' not inside brackets)
108 131
109 132 Note: We don't try to support multiple special assignment (a = b = %foo)
110 133 """
111 134 paren_level = 0
112 135 for i, ti in enumerate(token_line):
113 136 s = ti.string
114 137 if s == '=' and paren_level == 0:
115 138 return i
116 139 if s in {'(','[','{'}:
117 140 paren_level += 1
118 141 elif s in {')', ']', '}'}:
119 142 if paren_level > 0:
120 143 paren_level -= 1
121 144 return None
122 145
123 146 def find_end_of_continued_line(lines, start_line: int):
124 147 """Find the last line of a line explicitly extended using backslashes.
125 148
126 149 Uses 0-indexed line numbers.
127 150 """
128 151 end_line = start_line
129 152 while lines[end_line].endswith('\\\n'):
130 153 end_line += 1
131 154 if end_line >= len(lines):
132 155 break
133 156 return end_line
134 157
135 158 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
136 159 r"""Assemble a single line from multiple continued line pieces
137 160
138 161 Continued lines are lines ending in ``\``, and the line following the last
139 162 ``\`` in the block.
140 163
141 164 For example, this code continues over multiple lines::
142 165
143 166 if (assign_ix is not None) \
144 167 and (len(line) >= assign_ix + 2) \
145 168 and (line[assign_ix+1].string == '%') \
146 169 and (line[assign_ix+2].type == tokenize.NAME):
147 170
148 171 This statement contains four continued line pieces.
149 172 Assembling these pieces into a single line would give::
150 173
151 174 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
152 175
153 176 This uses 0-indexed line numbers. *start* is (lineno, colno).
154 177
155 178 Used to allow ``%magic`` and ``!system`` commands to be continued over
156 179 multiple lines.
157 180 """
158 181 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
159 182 return ' '.join([p.rstrip()[:-1] for p in parts[:-1]] # Strip backslash+newline
160 183 + [parts[-1].rstrip()]) # Strip newline from last line
161 184
162 185 class TokenTransformBase:
163 186 """Base class for transformations which examine tokens.
164 187
165 188 Special syntax should not be transformed when it occurs inside strings or
166 189 comments. This is hard to reliably avoid with regexes. The solution is to
167 190 tokenise the code as Python, and recognise the special syntax in the tokens.
168 191
169 192 IPython's special syntax is not valid Python syntax, so tokenising may go
170 193 wrong after the special syntax starts. These classes therefore find and
171 194 transform *one* instance of special syntax at a time into regular Python
172 195 syntax. After each transformation, tokens are regenerated to find the next
173 196 piece of special syntax.
174 197
175 198 Subclasses need to implement one class method (find)
176 199 and one regular method (transform).
177 200
178 201 The priority attribute can select which transformation to apply if multiple
179 202 transformers match in the same place. Lower numbers have higher priority.
180 203 This allows "%magic?" to be turned into a help call rather than a magic call.
181 204 """
182 205 # Lower numbers -> higher priority (for matches in the same location)
183 206 priority = 10
184 207
185 208 def sortby(self):
186 209 return self.start_line, self.start_col, self.priority
187 210
188 211 def __init__(self, start):
189 212 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
190 213 self.start_col = start[1]
191 214
192 215 @classmethod
193 216 def find(cls, tokens_by_line):
194 217 """Find one instance of special syntax in the provided tokens.
195 218
196 219 Tokens are grouped into logical lines for convenience,
197 220 so it is easy to e.g. look at the first token of each line.
198 221 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
199 222
200 223 This should return an instance of its class, pointing to the start
201 224 position it has found, or None if it found no match.
202 225 """
203 226 raise NotImplementedError
204 227
205 228 def transform(self, lines: List[str]):
206 229 """Transform one instance of special syntax found by ``find()``
207 230
208 231 Takes a list of strings representing physical lines,
209 232 returns a similar list of transformed lines.
210 233 """
211 234 raise NotImplementedError
212 235
213 236 class MagicAssign(TokenTransformBase):
214 237 """Transformer for assignments from magics (a = %foo)"""
215 238 @classmethod
216 239 def find(cls, tokens_by_line):
217 240 """Find the first magic assignment (a = %foo) in the cell.
218 241 """
219 242 for line in tokens_by_line:
220 243 assign_ix = _find_assign_op(line)
221 244 if (assign_ix is not None) \
222 245 and (len(line) >= assign_ix + 2) \
223 246 and (line[assign_ix+1].string == '%') \
224 247 and (line[assign_ix+2].type == tokenize.NAME):
225 248 return cls(line[assign_ix+1].start)
226 249
227 250 def transform(self, lines: List[str]):
228 251 """Transform a magic assignment found by the ``find()`` classmethod.
229 252 """
230 253 start_line, start_col = self.start_line, self.start_col
231 254 lhs = lines[start_line][:start_col]
232 255 end_line = find_end_of_continued_line(lines, start_line)
233 256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
234 257 assert rhs.startswith('%'), rhs
235 258 magic_name, _, args = rhs[1:].partition(' ')
236 259
237 260 lines_before = lines[:start_line]
238 261 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
239 262 new_line = lhs + call + '\n'
240 263 lines_after = lines[end_line+1:]
241 264
242 265 return lines_before + [new_line] + lines_after
243 266
244 267
245 268 class SystemAssign(TokenTransformBase):
246 269 """Transformer for assignments from system commands (a = !foo)"""
247 270 @classmethod
248 271 def find(cls, tokens_by_line):
249 272 """Find the first system assignment (a = !foo) in the cell.
250 273 """
251 274 for line in tokens_by_line:
252 275 assign_ix = _find_assign_op(line)
253 276 if (assign_ix is not None) \
254 277 and not line[assign_ix].line.strip().startswith('=') \
255 278 and (len(line) >= assign_ix + 2) \
256 279 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
257 280 ix = assign_ix + 1
258 281
259 282 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
260 283 if line[ix].string == '!':
261 284 return cls(line[ix].start)
262 285 elif not line[ix].string.isspace():
263 286 break
264 287 ix += 1
265 288
266 289 def transform(self, lines: List[str]):
267 290 """Transform a system assignment found by the ``find()`` classmethod.
268 291 """
269 292 start_line, start_col = self.start_line, self.start_col
270 293
271 294 lhs = lines[start_line][:start_col]
272 295 end_line = find_end_of_continued_line(lines, start_line)
273 296 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
274 297 assert rhs.startswith('!'), rhs
275 298 cmd = rhs[1:]
276 299
277 300 lines_before = lines[:start_line]
278 301 call = "get_ipython().getoutput({!r})".format(cmd)
279 302 new_line = lhs + call + '\n'
280 303 lines_after = lines[end_line + 1:]
281 304
282 305 return lines_before + [new_line] + lines_after
283 306
284 307 # The escape sequences that define the syntax transformations IPython will
285 308 # apply to user input. These can NOT be just changed here: many regular
286 309 # expressions and other parts of the code may use their hardcoded values, and
287 310 # for all intents and purposes they constitute the 'IPython syntax', so they
288 311 # should be considered fixed.
289 312
290 313 ESC_SHELL = '!' # Send line to underlying system shell
291 314 ESC_SH_CAP = '!!' # Send line to system shell and capture output
292 315 ESC_HELP = '?' # Find information about object
293 316 ESC_HELP2 = '??' # Find extra-detailed information about object
294 317 ESC_MAGIC = '%' # Call magic function
295 318 ESC_MAGIC2 = '%%' # Call cell-magic function
296 319 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
297 320 ESC_QUOTE2 = ';' # Quote all args as a single string, call
298 321 ESC_PAREN = '/' # Call first argument with rest of line as arguments
299 322
300 323 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
301 324 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
302 325
303 326 def _make_help_call(target, esc, next_input=None):
304 327 """Prepares a pinfo(2)/psearch call from a target name and the escape
305 328 (i.e. ? or ??)"""
306 329 method = 'pinfo2' if esc == '??' \
307 330 else 'psearch' if '*' in target \
308 331 else 'pinfo'
309 332 arg = " ".join([method, target])
310 333 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
311 334 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
312 335 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
313 336 if next_input is None:
314 337 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
315 338 else:
316 339 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
317 340 (next_input, t_magic_name, t_magic_arg_s)
318 341
319 342 def _tr_help(content):
320 343 """Translate lines escaped with: ?
321 344
322 345 A naked help line should fire the intro help screen (shell.show_usage())
323 346 """
324 347 if not content:
325 348 return 'get_ipython().show_usage()'
326 349
327 350 return _make_help_call(content, '?')
328 351
329 352 def _tr_help2(content):
330 353 """Translate lines escaped with: ??
331 354
332 355 A naked help line should fire the intro help screen (shell.show_usage())
333 356 """
334 357 if not content:
335 358 return 'get_ipython().show_usage()'
336 359
337 360 return _make_help_call(content, '??')
338 361
339 362 def _tr_magic(content):
340 363 "Translate lines escaped with a percent sign: %"
341 364 name, _, args = content.partition(' ')
342 365 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
343 366
344 367 def _tr_quote(content):
345 368 "Translate lines escaped with a comma: ,"
346 369 name, _, args = content.partition(' ')
347 370 return '%s("%s")' % (name, '", "'.join(args.split()) )
348 371
349 372 def _tr_quote2(content):
350 373 "Translate lines escaped with a semicolon: ;"
351 374 name, _, args = content.partition(' ')
352 375 return '%s("%s")' % (name, args)
353 376
354 377 def _tr_paren(content):
355 378 "Translate lines escaped with a slash: /"
356 379 name, _, args = content.partition(' ')
357 380 return '%s(%s)' % (name, ", ".join(args.split()))
358 381
359 382 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
360 383 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
361 384 ESC_HELP : _tr_help,
362 385 ESC_HELP2 : _tr_help2,
363 386 ESC_MAGIC : _tr_magic,
364 387 ESC_QUOTE : _tr_quote,
365 388 ESC_QUOTE2 : _tr_quote2,
366 389 ESC_PAREN : _tr_paren }
367 390
368 391 class EscapedCommand(TokenTransformBase):
369 392 """Transformer for escaped commands like %foo, !foo, or /foo"""
370 393 @classmethod
371 394 def find(cls, tokens_by_line):
372 395 """Find the first escaped command (%foo, !foo, etc.) in the cell.
373 396 """
374 397 for line in tokens_by_line:
375 398 if not line:
376 399 continue
377 400 ix = 0
378 401 ll = len(line)
379 402 while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
380 403 ix += 1
381 404 if ix >= ll:
382 405 continue
383 406 if line[ix].string in ESCAPE_SINGLES:
384 407 return cls(line[ix].start)
385 408
386 409 def transform(self, lines):
387 410 """Transform an escaped line found by the ``find()`` classmethod.
388 411 """
389 412 start_line, start_col = self.start_line, self.start_col
390 413
391 414 indent = lines[start_line][:start_col]
392 415 end_line = find_end_of_continued_line(lines, start_line)
393 416 line = assemble_continued_line(lines, (start_line, start_col), end_line)
394 417
395 418 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
396 419 escape, content = line[:2], line[2:]
397 420 else:
398 421 escape, content = line[:1], line[1:]
399 422
400 423 if escape in tr:
401 424 call = tr[escape](content)
402 425 else:
403 426 call = ''
404 427
405 428 lines_before = lines[:start_line]
406 429 new_line = indent + call + '\n'
407 430 lines_after = lines[end_line + 1:]
408 431
409 432 return lines_before + [new_line] + lines_after
410 433
411 434 _help_end_re = re.compile(r"""(%{0,2}
412 435 (?!\d)[\w*]+ # Variable name
413 436 (\.(?!\d)[\w*]+)* # .etc.etc
414 437 )
415 438 (\?\??)$ # ? or ??
416 439 """,
417 440 re.VERBOSE)
418 441
419 442 class HelpEnd(TokenTransformBase):
420 443 """Transformer for help syntax: obj? and obj??"""
421 444 # This needs to be higher priority (lower number) than EscapedCommand so
422 445 # that inspecting magics (%foo?) works.
423 446 priority = 5
424 447
425 448 def __init__(self, start, q_locn):
426 449 super().__init__(start)
427 450 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
428 451 self.q_col = q_locn[1]
429 452
430 453 @classmethod
431 454 def find(cls, tokens_by_line):
432 455 """Find the first help command (foo?) in the cell.
433 456 """
434 457 for line in tokens_by_line:
435 458 # Last token is NEWLINE; look at last but one
436 459 if len(line) > 2 and line[-2].string == '?':
437 460 # Find the first token that's not INDENT/DEDENT
438 461 ix = 0
439 462 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
440 463 ix += 1
441 464 return cls(line[ix].start, line[-2].start)
442 465
443 466 def transform(self, lines):
444 467 """Transform a help command found by the ``find()`` classmethod.
445 468 """
446 469 piece = ''.join(lines[self.start_line:self.q_line+1])
447 470 indent, content = piece[:self.start_col], piece[self.start_col:]
448 471 lines_before = lines[:self.start_line]
449 472 lines_after = lines[self.q_line + 1:]
450 473
451 474 m = _help_end_re.search(content)
452 475 if not m:
453 476 raise SyntaxError(content)
454 477 assert m is not None, content
455 478 target = m.group(1)
456 479 esc = m.group(3)
457 480
458 481 # If we're mid-command, put it back on the next prompt for the user.
459 482 next_input = None
460 483 if (not lines_before) and (not lines_after) \
461 484 and content.strip() != m.group(0):
462 485 next_input = content.rstrip('?\n')
463 486
464 487 call = _make_help_call(target, esc, next_input=next_input)
465 488 new_line = indent + call + '\n'
466 489
467 490 return lines_before + [new_line] + lines_after
468 491
469 492 def make_tokens_by_line(lines:List[str]):
470 493 """Tokenize a series of lines and group tokens by line.
471 494
472 495 The tokens for a multiline Python string or expression are grouped as one
473 496 line. All lines except the last lines should keep their line ending ('\\n',
474 497 '\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
475 498 for example when passing block of text to this function.
476 499
477 500 """
478 501 # NL tokens are used inside multiline expressions, but also after blank
479 502 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
480 503 # We want to group the former case together but split the latter, so we
481 504 # track parentheses level, similar to the internals of tokenize.
482 505
483 506 # reexported from token on 3.7+
484 507 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
485 508 tokens_by_line:List[List[Any]] = [[]]
486 509 if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
487 510 warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
488 511 parenlev = 0
489 512 try:
490 513 for token in tokenize.generate_tokens(iter(lines).__next__):
491 514 tokens_by_line[-1].append(token)
492 515 if (token.type == NEWLINE) \
493 516 or ((token.type == NL) and (parenlev <= 0)):
494 517 tokens_by_line.append([])
495 518 elif token.string in {'(', '[', '{'}:
496 519 parenlev += 1
497 520 elif token.string in {')', ']', '}'}:
498 521 if parenlev > 0:
499 522 parenlev -= 1
500 523 except tokenize.TokenError:
501 524 # Input ended in a multiline string or expression. That's OK for us.
502 525 pass
503 526
504 527
505 528 if not tokens_by_line[-1]:
506 529 tokens_by_line.pop()
507 530
508 531
509 532 return tokens_by_line
510 533
511 534
512 535 def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
513 536 """Check if the depth of brackets in the list of tokens drops below 0"""
514 537 parenlev = 0
515 538 for token in tokens:
516 539 if token.string in {"(", "[", "{"}:
517 540 parenlev += 1
518 541 elif token.string in {")", "]", "}"}:
519 542 parenlev -= 1
520 543 if parenlev < 0:
521 544 return True
522 545 return False
523 546
524 547
525 548 def show_linewise_tokens(s: str):
526 549 """For investigation and debugging"""
527 550 if not s.endswith('\n'):
528 551 s += '\n'
529 552 lines = s.splitlines(keepends=True)
530 553 for line in make_tokens_by_line(lines):
531 554 print("Line -------")
532 555 for tokinfo in line:
533 556 print(" ", tokinfo)
534 557
535 558 # Arbitrary limit to prevent getting stuck in infinite loops
536 559 TRANSFORM_LOOP_LIMIT = 500
537 560
538 561 class TransformerManager:
539 562 """Applies various transformations to a cell or code block.
540 563
541 564 The key methods for external use are ``transform_cell()``
542 565 and ``check_complete()``.
543 566 """
544 567 def __init__(self):
545 568 self.cleanup_transforms = [
546 569 leading_empty_lines,
547 570 leading_indent,
548 571 classic_prompt,
549 572 ipython_prompt,
550 573 ]
551 574 self.line_transforms = [
552 575 cell_magic,
553 576 ]
554 577 self.token_transformers = [
555 578 MagicAssign,
556 579 SystemAssign,
557 580 EscapedCommand,
558 581 HelpEnd,
559 582 ]
560 583
561 584 def do_one_token_transform(self, lines):
562 585 """Find and run the transform earliest in the code.
563 586
564 587 Returns (changed, lines).
565 588
566 589 This method is called repeatedly until changed is False, indicating
567 590 that all available transformations are complete.
568 591
569 592 The tokens following IPython special syntax might not be valid, so
570 593 the transformed code is retokenised every time to identify the next
571 594 piece of special syntax. Hopefully long code cells are mostly valid
572 595 Python, not using lots of IPython special syntax, so this shouldn't be
573 596 a performance issue.
574 597 """
575 598 tokens_by_line = make_tokens_by_line(lines)
576 599 candidates = []
577 600 for transformer_cls in self.token_transformers:
578 601 transformer = transformer_cls.find(tokens_by_line)
579 602 if transformer:
580 603 candidates.append(transformer)
581 604
582 605 if not candidates:
583 606 # Nothing to transform
584 607 return False, lines
585 608 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
586 609 for transformer in ordered_transformers:
587 610 try:
588 611 return True, transformer.transform(lines)
589 612 except SyntaxError:
590 613 pass
591 614 return False, lines
592 615
593 616 def do_token_transforms(self, lines):
594 617 for _ in range(TRANSFORM_LOOP_LIMIT):
595 618 changed, lines = self.do_one_token_transform(lines)
596 619 if not changed:
597 620 return lines
598 621
599 622 raise RuntimeError("Input transformation still changing after "
600 623 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
601 624
602 625 def transform_cell(self, cell: str) -> str:
603 626 """Transforms a cell of input code"""
604 627 if not cell.endswith('\n'):
605 628 cell += '\n' # Ensure the cell has a trailing newline
606 629 lines = cell.splitlines(keepends=True)
607 630 for transform in self.cleanup_transforms + self.line_transforms:
608 631 lines = transform(lines)
609 632
610 633 lines = self.do_token_transforms(lines)
611 634 return ''.join(lines)
612 635
613 636 def check_complete(self, cell: str):
614 637 """Return whether a block of code is ready to execute, or should be continued
615 638
616 639 Parameters
617 640 ----------
618 641 source : string
619 642 Python input code, which can be multiline.
620 643
621 644 Returns
622 645 -------
623 646 status : str
624 647 One of 'complete', 'incomplete', or 'invalid' if source is not a
625 648 prefix of valid code.
626 649 indent_spaces : int or None
627 650 The number of spaces by which to indent the next line of code. If
628 651 status is not 'incomplete', this is None.
629 652 """
630 653 # Remember if the lines ends in a new line.
631 654 ends_with_newline = False
632 655 for character in reversed(cell):
633 656 if character == '\n':
634 657 ends_with_newline = True
635 658 break
636 659 elif character.strip():
637 660 break
638 661 else:
639 662 continue
640 663
641 664 if not ends_with_newline:
642 665 # Append an newline for consistent tokenization
643 666 # See https://bugs.python.org/issue33899
644 667 cell += '\n'
645 668
646 669 lines = cell.splitlines(keepends=True)
647 670
648 671 if not lines:
649 672 return 'complete', None
650 673
651 674 if lines[-1].endswith('\\'):
652 675 # Explicit backslash continuation
653 676 return 'incomplete', find_last_indent(lines)
654 677
655 678 try:
656 679 for transform in self.cleanup_transforms:
657 680 if not getattr(transform, 'has_side_effects', False):
658 681 lines = transform(lines)
659 682 except SyntaxError:
660 683 return 'invalid', None
661 684
662 685 if lines[0].startswith('%%'):
663 686 # Special case for cell magics - completion marked by blank line
664 687 if lines[-1].strip():
665 688 return 'incomplete', find_last_indent(lines)
666 689 else:
667 690 return 'complete', None
668 691
669 692 try:
670 693 for transform in self.line_transforms:
671 694 if not getattr(transform, 'has_side_effects', False):
672 695 lines = transform(lines)
673 696 lines = self.do_token_transforms(lines)
674 697 except SyntaxError:
675 698 return 'invalid', None
676 699
677 700 tokens_by_line = make_tokens_by_line(lines)
678 701
679 702 # Bail if we got one line and there are more closing parentheses than
680 703 # the opening ones
681 704 if (
682 705 len(lines) == 1
683 706 and tokens_by_line
684 707 and has_sunken_brackets(tokens_by_line[0])
685 708 ):
686 709 return "invalid", None
687 710
688 711 if not tokens_by_line:
689 712 return 'incomplete', find_last_indent(lines)
690 713
691 714 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
692 715 # We're in a multiline string or expression
693 716 return 'incomplete', find_last_indent(lines)
694 717
695 718 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
696 719
697 720 # Pop the last line which only contains DEDENTs and ENDMARKER
698 721 last_token_line = None
699 722 if {t.type for t in tokens_by_line[-1]} in [
700 723 {tokenize.DEDENT, tokenize.ENDMARKER},
701 724 {tokenize.ENDMARKER}
702 725 ] and len(tokens_by_line) > 1:
703 726 last_token_line = tokens_by_line.pop()
704 727
705 728 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
706 729 tokens_by_line[-1].pop()
707 730
708 731 if not tokens_by_line[-1]:
709 732 return 'incomplete', find_last_indent(lines)
710 733
711 734 if tokens_by_line[-1][-1].string == ':':
712 735 # The last line starts a block (e.g. 'if foo:')
713 736 ix = 0
714 737 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
715 738 ix += 1
716 739
717 740 indent = tokens_by_line[-1][ix].start[1]
718 741 return 'incomplete', indent + 4
719 742
720 743 if tokens_by_line[-1][0].line.endswith('\\'):
721 744 return 'incomplete', None
722 745
723 746 # At this point, our checks think the code is complete (or invalid).
724 747 # We'll use codeop.compile_command to check this with the real parser
725 748 try:
726 749 with warnings.catch_warnings():
727 750 warnings.simplefilter('error', SyntaxWarning)
728 751 res = compile_command(''.join(lines), symbol='exec')
729 752 except (SyntaxError, OverflowError, ValueError, TypeError,
730 753 MemoryError, SyntaxWarning):
731 754 return 'invalid', None
732 755 else:
733 756 if res is None:
734 757 return 'incomplete', find_last_indent(lines)
735 758
736 759 if last_token_line and last_token_line[0].type == tokenize.DEDENT:
737 760 if ends_with_newline:
738 761 return 'complete', None
739 762 return 'incomplete', find_last_indent(lines)
740 763
741 764 # If there's a blank line at the end, assume we're ready to execute
742 765 if not lines[-1].strip():
743 766 return 'complete', None
744 767
745 768 return 'complete', None
746 769
747 770
748 771 def find_last_indent(lines):
749 772 m = _indent_re.match(lines[-1])
750 773 if not m:
751 774 return 0
752 775 return len(m.group(0).replace('\t', ' '*4))
@@ -1,126 +1,166 b''
1 1 """Tests for the line-based transformers in IPython.core.inputtransformer2
2 2
3 3 Line-based transformers are the simpler ones; token-based transformers are
4 4 more complex. See test_inputtransformer2 for tests for token-based transformers.
5 5 """
6 6 import nose.tools as nt
7 7
8 8 from IPython.core import inputtransformer2 as ipt2
9 9
10 10 CELL_MAGIC = ("""\
11 11 %%foo arg
12 12 body 1
13 13 body 2
14 14 """, """\
15 15 get_ipython().run_cell_magic('foo', 'arg', 'body 1\\nbody 2\\n')
16 16 """)
17 17
18 18 def test_cell_magic():
19 19 for sample, expected in [CELL_MAGIC]:
20 20 nt.assert_equal(ipt2.cell_magic(sample.splitlines(keepends=True)),
21 21 expected.splitlines(keepends=True))
22 22
23 23 CLASSIC_PROMPT = ("""\
24 24 >>> for a in range(5):
25 25 ... print(a)
26 26 """, """\
27 27 for a in range(5):
28 28 print(a)
29 29 """)
30 30
31 31 CLASSIC_PROMPT_L2 = ("""\
32 32 for a in range(5):
33 33 ... print(a)
34 34 ... print(a ** 2)
35 35 """, """\
36 36 for a in range(5):
37 37 print(a)
38 38 print(a ** 2)
39 39 """)
40 40
41 41 def test_classic_prompt():
42 42 for sample, expected in [CLASSIC_PROMPT, CLASSIC_PROMPT_L2]:
43 43 nt.assert_equal(ipt2.classic_prompt(sample.splitlines(keepends=True)),
44 44 expected.splitlines(keepends=True))
45 45
46 46 IPYTHON_PROMPT = ("""\
47 47 In [1]: for a in range(5):
48 48 ...: print(a)
49 49 """, """\
50 50 for a in range(5):
51 51 print(a)
52 52 """)
53 53
54 54 IPYTHON_PROMPT_L2 = ("""\
55 55 for a in range(5):
56 56 ...: print(a)
57 57 ...: print(a ** 2)
58 58 """, """\
59 59 for a in range(5):
60 60 print(a)
61 61 print(a ** 2)
62 62 """)
63 63
64
65 IPYTHON_PROMPT_VI_INS = (
66 """\
67 [ins] In [11]: def a():
68 ...: 123
69 ...:
70 ...: 123
71 """,
72 """\
73 def a():
74 123
75
76 123
77 """,
78 )
79
80 IPYTHON_PROMPT_VI_NAV = (
81 """\
82 [nav] In [11]: def a():
83 ...: 123
84 ...:
85 ...: 123
86 """,
87 """\
88 def a():
89 123
90
91 123
92 """,
93 )
94
95
64 96 def test_ipython_prompt():
65 for sample, expected in [IPYTHON_PROMPT, IPYTHON_PROMPT_L2]:
66 nt.assert_equal(ipt2.ipython_prompt(sample.splitlines(keepends=True)),
67 expected.splitlines(keepends=True))
97 for sample, expected in [
98 IPYTHON_PROMPT,
99 IPYTHON_PROMPT_L2,
100 IPYTHON_PROMPT_VI_INS,
101 IPYTHON_PROMPT_VI_NAV,
102 ]:
103 nt.assert_equal(
104 ipt2.ipython_prompt(sample.splitlines(keepends=True)),
105 expected.splitlines(keepends=True),
106 )
107
68 108
69 109 INDENT_SPACES = ("""\
70 110 if True:
71 111 a = 3
72 112 """, """\
73 113 if True:
74 114 a = 3
75 115 """)
76 116
77 117 INDENT_TABS = ("""\
78 118 \tif True:
79 119 \t\tb = 4
80 120 """, """\
81 121 if True:
82 122 \tb = 4
83 123 """)
84 124
85 125 def test_leading_indent():
86 126 for sample, expected in [INDENT_SPACES, INDENT_TABS]:
87 127 nt.assert_equal(ipt2.leading_indent(sample.splitlines(keepends=True)),
88 128 expected.splitlines(keepends=True))
89 129
90 130 LEADING_EMPTY_LINES = ("""\
91 131 \t
92 132
93 133 if True:
94 134 a = 3
95 135
96 136 b = 4
97 137 """, """\
98 138 if True:
99 139 a = 3
100 140
101 141 b = 4
102 142 """)
103 143
104 144 ONLY_EMPTY_LINES = ("""\
105 145 \t
106 146
107 147 """, """\
108 148 \t
109 149
110 150 """)
111 151
112 152 def test_leading_empty_lines():
113 153 for sample, expected in [LEADING_EMPTY_LINES, ONLY_EMPTY_LINES]:
114 154 nt.assert_equal(
115 155 ipt2.leading_empty_lines(sample.splitlines(keepends=True)),
116 156 expected.splitlines(keepends=True))
117 157
118 158 CRLF_MAGIC = ([
119 159 "%%ls\r\n"
120 160 ], [
121 161 "get_ipython().run_cell_magic('ls', '', '')\n"
122 162 ])
123 163
124 164 def test_crlf_magic():
125 165 for sample, expected in [CRLF_MAGIC]:
126 166 nt.assert_equal(ipt2.cell_magic(sample), expected)
General Comments 0
You need to be logged in to leave comments. Login now