##// END OF EJS Templates
Remove a condition from check_complete...
Tony Fast -
Show More
@@ -1,698 +1,694
1 1 """Input transformer machinery to support IPython special syntax.
2 2
3 3 This includes the machinery to recognise and transform ``%magic`` commands,
4 4 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
5 5
6 6 Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
7 7 deprecated in 7.0.
8 8 """
9 9
10 10 # Copyright (c) IPython Development Team.
11 11 # Distributed under the terms of the Modified BSD License.
12 12
13 13 from codeop import compile_command
14 14 import re
15 15 import tokenize
16 16 from typing import List, Tuple
17 17 import warnings
18 18
19 19 _indent_re = re.compile(r'^[ \t]+')
20 20
21 21 def leading_indent(lines):
22 22 """Remove leading indentation.
23 23
24 24 If the first line starts with a spaces or tabs, the same whitespace will be
25 25 removed from each following line in the cell.
26 26 """
27 27 if not lines:
28 28 return lines
29 29 m = _indent_re.match(lines[0])
30 30 if not m:
31 31 return lines
32 32 space = m.group(0)
33 33 n = len(space)
34 34 return [l[n:] if l.startswith(space) else l
35 35 for l in lines]
36 36
37 37 class PromptStripper:
38 38 """Remove matching input prompts from a block of input.
39 39
40 40 Parameters
41 41 ----------
42 42 prompt_re : regular expression
43 43 A regular expression matching any input prompt (including continuation,
44 44 e.g. ``...``)
45 45 initial_re : regular expression, optional
46 46 A regular expression matching only the initial prompt, but not continuation.
47 47 If no initial expression is given, prompt_re will be used everywhere.
48 48 Used mainly for plain Python prompts (``>>>``), where the continuation prompt
49 49 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
50 50
51 51 If initial_re and prompt_re differ,
52 52 only initial_re will be tested against the first line.
53 53 If any prompt is found on the first two lines,
54 54 prompts will be stripped from the rest of the block.
55 55 """
56 56 def __init__(self, prompt_re, initial_re=None):
57 57 self.prompt_re = prompt_re
58 58 self.initial_re = initial_re or prompt_re
59 59
60 60 def _strip(self, lines):
61 61 return [self.prompt_re.sub('', l, count=1) for l in lines]
62 62
63 63 def __call__(self, lines):
64 64 if not lines:
65 65 return lines
66 66 if self.initial_re.match(lines[0]) or \
67 67 (len(lines) > 1 and self.prompt_re.match(lines[1])):
68 68 return self._strip(lines)
69 69 return lines
70 70
71 71 classic_prompt = PromptStripper(
72 72 prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
73 73 initial_re=re.compile(r'^>>>( |$)')
74 74 )
75 75
76 76 ipython_prompt = PromptStripper(re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)'))
77 77
78 78 def cell_magic(lines):
79 79 if not lines or not lines[0].startswith('%%'):
80 80 return lines
81 81 if re.match('%%\w+\?', lines[0]):
82 82 # This case will be handled by help_end
83 83 return lines
84 84 magic_name, _, first_line = lines[0][2:-1].partition(' ')
85 85 body = ''.join(lines[1:])
86 86 return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
87 87 % (magic_name, first_line, body)]
88 88
89 89
90 90 def _find_assign_op(token_line):
91 91 """Get the index of the first assignment in the line ('=' not inside brackets)
92 92
93 93 Note: We don't try to support multiple special assignment (a = b = %foo)
94 94 """
95 95 paren_level = 0
96 96 for i, ti in enumerate(token_line):
97 97 s = ti.string
98 98 if s == '=' and paren_level == 0:
99 99 return i
100 100 if s in '([{':
101 101 paren_level += 1
102 102 elif s in ')]}':
103 103 if paren_level > 0:
104 104 paren_level -= 1
105 105
106 106 def find_end_of_continued_line(lines, start_line: int):
107 107 """Find the last line of a line explicitly extended using backslashes.
108 108
109 109 Uses 0-indexed line numbers.
110 110 """
111 111 end_line = start_line
112 112 while lines[end_line].endswith('\\\n'):
113 113 end_line += 1
114 114 if end_line >= len(lines):
115 115 break
116 116 return end_line
117 117
118 118 def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
119 119 """Assemble a single line from multiple continued line pieces
120 120
121 121 Continued lines are lines ending in ``\``, and the line following the last
122 122 ``\`` in the block.
123 123
124 124 For example, this code continues over multiple lines::
125 125
126 126 if (assign_ix is not None) \
127 127 and (len(line) >= assign_ix + 2) \
128 128 and (line[assign_ix+1].string == '%') \
129 129 and (line[assign_ix+2].type == tokenize.NAME):
130 130
131 131 This statement contains four continued line pieces.
132 132 Assembling these pieces into a single line would give::
133 133
134 134 if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
135 135
136 136 This uses 0-indexed line numbers. *start* is (lineno, colno).
137 137
138 138 Used to allow ``%magic`` and ``!system`` commands to be continued over
139 139 multiple lines.
140 140 """
141 141 parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
142 142 return ' '.join([p[:-2] for p in parts[:-1]] # Strip backslash+newline
143 143 + [parts[-1][:-1]]) # Strip newline from last line
144 144
145 145 class TokenTransformBase:
146 146 """Base class for transformations which examine tokens.
147 147
148 148 Special syntax should not be transformed when it occurs inside strings or
149 149 comments. This is hard to reliably avoid with regexes. The solution is to
150 150 tokenise the code as Python, and recognise the special syntax in the tokens.
151 151
152 152 IPython's special syntax is not valid Python syntax, so tokenising may go
153 153 wrong after the special syntax starts. These classes therefore find and
154 154 transform *one* instance of special syntax at a time into regular Python
155 155 syntax. After each transformation, tokens are regenerated to find the next
156 156 piece of special syntax.
157 157
158 158 Subclasses need to implement one class method (find)
159 159 and one regular method (transform).
160 160
161 161 The priority attribute can select which transformation to apply if multiple
162 162 transformers match in the same place. Lower numbers have higher priority.
163 163 This allows "%magic?" to be turned into a help call rather than a magic call.
164 164 """
165 165 # Lower numbers -> higher priority (for matches in the same location)
166 166 priority = 10
167 167
168 168 def sortby(self):
169 169 return self.start_line, self.start_col, self.priority
170 170
171 171 def __init__(self, start):
172 172 self.start_line = start[0] - 1 # Shift from 1-index to 0-index
173 173 self.start_col = start[1]
174 174
175 175 @classmethod
176 176 def find(cls, tokens_by_line):
177 177 """Find one instance of special syntax in the provided tokens.
178 178
179 179 Tokens are grouped into logical lines for convenience,
180 180 so it is easy to e.g. look at the first token of each line.
181 181 *tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
182 182
183 183 This should return an instance of its class, pointing to the start
184 184 position it has found, or None if it found no match.
185 185 """
186 186 raise NotImplementedError
187 187
188 188 def transform(self, lines: List[str]):
189 189 """Transform one instance of special syntax found by ``find()``
190 190
191 191 Takes a list of strings representing physical lines,
192 192 returns a similar list of transformed lines.
193 193 """
194 194 raise NotImplementedError
195 195
196 196 class MagicAssign(TokenTransformBase):
197 197 """Transformer for assignments from magics (a = %foo)"""
198 198 @classmethod
199 199 def find(cls, tokens_by_line):
200 200 """Find the first magic assignment (a = %foo) in the cell.
201 201 """
202 202 for line in tokens_by_line:
203 203 assign_ix = _find_assign_op(line)
204 204 if (assign_ix is not None) \
205 205 and (len(line) >= assign_ix + 2) \
206 206 and (line[assign_ix+1].string == '%') \
207 207 and (line[assign_ix+2].type == tokenize.NAME):
208 208 return cls(line[assign_ix+1].start)
209 209
210 210 def transform(self, lines: List[str]):
211 211 """Transform a magic assignment found by the ``find()`` classmethod.
212 212 """
213 213 start_line, start_col = self.start_line, self.start_col
214 214 lhs = lines[start_line][:start_col]
215 215 end_line = find_end_of_continued_line(lines, start_line)
216 216 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
217 217 assert rhs.startswith('%'), rhs
218 218 magic_name, _, args = rhs[1:].partition(' ')
219 219
220 220 lines_before = lines[:start_line]
221 221 call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
222 222 new_line = lhs + call + '\n'
223 223 lines_after = lines[end_line+1:]
224 224
225 225 return lines_before + [new_line] + lines_after
226 226
227 227
228 228 class SystemAssign(TokenTransformBase):
229 229 """Transformer for assignments from system commands (a = !foo)"""
230 230 @classmethod
231 231 def find(cls, tokens_by_line):
232 232 """Find the first system assignment (a = !foo) in the cell.
233 233 """
234 234 for line in tokens_by_line:
235 235 assign_ix = _find_assign_op(line)
236 236 if (assign_ix is not None) \
237 237 and not line[assign_ix].line.strip().startswith('=') \
238 238 and (len(line) >= assign_ix + 2) \
239 239 and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
240 240 ix = assign_ix + 1
241 241
242 242 while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
243 243 if line[ix].string == '!':
244 244 return cls(line[ix].start)
245 245 elif not line[ix].string.isspace():
246 246 break
247 247 ix += 1
248 248
249 249 def transform(self, lines: List[str]):
250 250 """Transform a system assignment found by the ``find()`` classmethod.
251 251 """
252 252 start_line, start_col = self.start_line, self.start_col
253 253
254 254 lhs = lines[start_line][:start_col]
255 255 end_line = find_end_of_continued_line(lines, start_line)
256 256 rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
257 257 assert rhs.startswith('!'), rhs
258 258 cmd = rhs[1:]
259 259
260 260 lines_before = lines[:start_line]
261 261 call = "get_ipython().getoutput({!r})".format(cmd)
262 262 new_line = lhs + call + '\n'
263 263 lines_after = lines[end_line + 1:]
264 264
265 265 return lines_before + [new_line] + lines_after
266 266
267 267 # The escape sequences that define the syntax transformations IPython will
268 268 # apply to user input. These can NOT be just changed here: many regular
269 269 # expressions and other parts of the code may use their hardcoded values, and
270 270 # for all intents and purposes they constitute the 'IPython syntax', so they
271 271 # should be considered fixed.
272 272
273 273 ESC_SHELL = '!' # Send line to underlying system shell
274 274 ESC_SH_CAP = '!!' # Send line to system shell and capture output
275 275 ESC_HELP = '?' # Find information about object
276 276 ESC_HELP2 = '??' # Find extra-detailed information about object
277 277 ESC_MAGIC = '%' # Call magic function
278 278 ESC_MAGIC2 = '%%' # Call cell-magic function
279 279 ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
280 280 ESC_QUOTE2 = ';' # Quote all args as a single string, call
281 281 ESC_PAREN = '/' # Call first argument with rest of line as arguments
282 282
283 283 ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
284 284 ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
285 285
286 286 def _make_help_call(target, esc, next_input=None):
287 287 """Prepares a pinfo(2)/psearch call from a target name and the escape
288 288 (i.e. ? or ??)"""
289 289 method = 'pinfo2' if esc == '??' \
290 290 else 'psearch' if '*' in target \
291 291 else 'pinfo'
292 292 arg = " ".join([method, target])
293 293 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
294 294 t_magic_name, _, t_magic_arg_s = arg.partition(' ')
295 295 t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
296 296 if next_input is None:
297 297 return 'get_ipython().run_line_magic(%r, %r)' % (t_magic_name, t_magic_arg_s)
298 298 else:
299 299 return 'get_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
300 300 (next_input, t_magic_name, t_magic_arg_s)
301 301
302 302 def _tr_help(content):
303 303 """Translate lines escaped with: ?
304 304
305 305 A naked help line should fire the intro help screen (shell.show_usage())
306 306 """
307 307 if not content:
308 308 return 'get_ipython().show_usage()'
309 309
310 310 return _make_help_call(content, '?')
311 311
312 312 def _tr_help2(content):
313 313 """Translate lines escaped with: ??
314 314
315 315 A naked help line should fire the intro help screen (shell.show_usage())
316 316 """
317 317 if not content:
318 318 return 'get_ipython().show_usage()'
319 319
320 320 return _make_help_call(content, '??')
321 321
322 322 def _tr_magic(content):
323 323 "Translate lines escaped with a percent sign: %"
324 324 name, _, args = content.partition(' ')
325 325 return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
326 326
327 327 def _tr_quote(content):
328 328 "Translate lines escaped with a comma: ,"
329 329 name, _, args = content.partition(' ')
330 330 return '%s("%s")' % (name, '", "'.join(args.split()) )
331 331
332 332 def _tr_quote2(content):
333 333 "Translate lines escaped with a semicolon: ;"
334 334 name, _, args = content.partition(' ')
335 335 return '%s("%s")' % (name, args)
336 336
337 337 def _tr_paren(content):
338 338 "Translate lines escaped with a slash: /"
339 339 name, _, args = content.partition(' ')
340 340 return '%s(%s)' % (name, ", ".join(args.split()))
341 341
342 342 tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
343 343 ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
344 344 ESC_HELP : _tr_help,
345 345 ESC_HELP2 : _tr_help2,
346 346 ESC_MAGIC : _tr_magic,
347 347 ESC_QUOTE : _tr_quote,
348 348 ESC_QUOTE2 : _tr_quote2,
349 349 ESC_PAREN : _tr_paren }
350 350
351 351 class EscapedCommand(TokenTransformBase):
352 352 """Transformer for escaped commands like %foo, !foo, or /foo"""
353 353 @classmethod
354 354 def find(cls, tokens_by_line):
355 355 """Find the first escaped command (%foo, !foo, etc.) in the cell.
356 356 """
357 357 for line in tokens_by_line:
358 358 ix = 0
359 359 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
360 360 ix += 1
361 361 if line[ix].string in ESCAPE_SINGLES:
362 362 return cls(line[ix].start)
363 363
364 364 def transform(self, lines):
365 365 """Transform an escaped line found by the ``find()`` classmethod.
366 366 """
367 367 start_line, start_col = self.start_line, self.start_col
368 368
369 369 indent = lines[start_line][:start_col]
370 370 end_line = find_end_of_continued_line(lines, start_line)
371 371 line = assemble_continued_line(lines, (start_line, start_col), end_line)
372 372
373 373 if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
374 374 escape, content = line[:2], line[2:]
375 375 else:
376 376 escape, content = line[:1], line[1:]
377 377
378 378 if escape in tr:
379 379 call = tr[escape](content)
380 380 else:
381 381 call = ''
382 382
383 383 lines_before = lines[:start_line]
384 384 new_line = indent + call + '\n'
385 385 lines_after = lines[end_line + 1:]
386 386
387 387 return lines_before + [new_line] + lines_after
388 388
389 389 _help_end_re = re.compile(r"""(%{0,2}
390 390 [a-zA-Z_*][\w*]* # Variable name
391 391 (\.[a-zA-Z_*][\w*]*)* # .etc.etc
392 392 )
393 393 (\?\??)$ # ? or ??
394 394 """,
395 395 re.VERBOSE)
396 396
397 397 class HelpEnd(TokenTransformBase):
398 398 """Transformer for help syntax: obj? and obj??"""
399 399 # This needs to be higher priority (lower number) than EscapedCommand so
400 400 # that inspecting magics (%foo?) works.
401 401 priority = 5
402 402
403 403 def __init__(self, start, q_locn):
404 404 super().__init__(start)
405 405 self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
406 406 self.q_col = q_locn[1]
407 407
408 408 @classmethod
409 409 def find(cls, tokens_by_line):
410 410 """Find the first help command (foo?) in the cell.
411 411 """
412 412 for line in tokens_by_line:
413 413 # Last token is NEWLINE; look at last but one
414 414 if len(line) > 2 and line[-2].string == '?':
415 415 # Find the first token that's not INDENT/DEDENT
416 416 ix = 0
417 417 while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
418 418 ix += 1
419 419 return cls(line[ix].start, line[-2].start)
420 420
421 421 def transform(self, lines):
422 422 """Transform a help command found by the ``find()`` classmethod.
423 423 """
424 424 piece = ''.join(lines[self.start_line:self.q_line+1])
425 425 indent, content = piece[:self.start_col], piece[self.start_col:]
426 426 lines_before = lines[:self.start_line]
427 427 lines_after = lines[self.q_line + 1:]
428 428
429 429 m = _help_end_re.search(content)
430 430 if not m:
431 431 raise SyntaxError(content)
432 432 assert m is not None, content
433 433 target = m.group(1)
434 434 esc = m.group(3)
435 435
436 436 # If we're mid-command, put it back on the next prompt for the user.
437 437 next_input = None
438 438 if (not lines_before) and (not lines_after) \
439 439 and content.strip() != m.group(0):
440 440 next_input = content.rstrip('?\n')
441 441
442 442 call = _make_help_call(target, esc, next_input=next_input)
443 443 new_line = indent + call + '\n'
444 444
445 445 return lines_before + [new_line] + lines_after
446 446
447 447 def make_tokens_by_line(lines):
448 448 """Tokenize a series of lines and group tokens by line.
449 449
450 450 The tokens for a multiline Python string or expression are
451 451 grouped as one line.
452 452 """
453 453 # NL tokens are used inside multiline expressions, but also after blank
454 454 # lines or comments. This is intentional - see https://bugs.python.org/issue17061
455 455 # We want to group the former case together but split the latter, so we
456 456 # track parentheses level, similar to the internals of tokenize.
457 457 NEWLINE, NL = tokenize.NEWLINE, tokenize.NL
458 458 tokens_by_line = [[]]
459 459 parenlev = 0
460 460 try:
461 461 for token in tokenize.generate_tokens(iter(lines).__next__):
462 462 tokens_by_line[-1].append(token)
463 463 if (token.type == NEWLINE) \
464 464 or ((token.type == NL) and (parenlev <= 0)):
465 465 tokens_by_line.append([])
466 466 elif token.string in {'(', '[', '{'}:
467 467 parenlev += 1
468 468 elif token.string in {')', ']', '}'}:
469 469 if parenlev > 0:
470 470 parenlev -= 1
471 471 except tokenize.TokenError:
472 472 # Input ended in a multiline string or expression. That's OK for us.
473 473 pass
474 474
475 475
476 476 if not tokens_by_line[-1]:
477 477 tokens_by_line.pop()
478 478
479 479
480 480 return tokens_by_line
481 481
482 482 def show_linewise_tokens(s: str):
483 483 """For investigation and debugging"""
484 484 if not s.endswith('\n'):
485 485 s += '\n'
486 486 lines = s.splitlines(keepends=True)
487 487 for line in make_tokens_by_line(lines):
488 488 print("Line -------")
489 489 for tokinfo in line:
490 490 print(" ", tokinfo)
491 491
492 492 # Arbitrary limit to prevent getting stuck in infinite loops
493 493 TRANSFORM_LOOP_LIMIT = 500
494 494
495 495 class TransformerManager:
496 496 """Applies various transformations to a cell or code block.
497 497
498 498 The key methods for external use are ``transform_cell()``
499 499 and ``check_complete()``.
500 500 """
501 501 def __init__(self):
502 502 self.cleanup_transforms = [
503 503 leading_indent,
504 504 classic_prompt,
505 505 ipython_prompt,
506 506 ]
507 507 self.line_transforms = [
508 508 cell_magic,
509 509 ]
510 510 self.token_transformers = [
511 511 MagicAssign,
512 512 SystemAssign,
513 513 EscapedCommand,
514 514 HelpEnd,
515 515 ]
516 516
517 517 def do_one_token_transform(self, lines):
518 518 """Find and run the transform earliest in the code.
519 519
520 520 Returns (changed, lines).
521 521
522 522 This method is called repeatedly until changed is False, indicating
523 523 that all available transformations are complete.
524 524
525 525 The tokens following IPython special syntax might not be valid, so
526 526 the transformed code is retokenised every time to identify the next
527 527 piece of special syntax. Hopefully long code cells are mostly valid
528 528 Python, not using lots of IPython special syntax, so this shouldn't be
529 529 a performance issue.
530 530 """
531 531 tokens_by_line = make_tokens_by_line(lines)
532 532 candidates = []
533 533 for transformer_cls in self.token_transformers:
534 534 transformer = transformer_cls.find(tokens_by_line)
535 535 if transformer:
536 536 candidates.append(transformer)
537 537
538 538 if not candidates:
539 539 # Nothing to transform
540 540 return False, lines
541 541 ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
542 542 for transformer in ordered_transformers:
543 543 try:
544 544 return True, transformer.transform(lines)
545 545 except SyntaxError:
546 546 pass
547 547 return False, lines
548 548
549 549 def do_token_transforms(self, lines):
550 550 for _ in range(TRANSFORM_LOOP_LIMIT):
551 551 changed, lines = self.do_one_token_transform(lines)
552 552 if not changed:
553 553 return lines
554 554
555 555 raise RuntimeError("Input transformation still changing after "
556 556 "%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
557 557
558 558 def transform_cell(self, cell: str) -> str:
559 559 """Transforms a cell of input code"""
560 560 if not cell.endswith('\n'):
561 561 cell += '\n' # Ensure the cell has a trailing newline
562 562 lines = cell.splitlines(keepends=True)
563 563 for transform in self.cleanup_transforms + self.line_transforms:
564 564 lines = transform(lines)
565 565
566 566 lines = self.do_token_transforms(lines)
567 567 return ''.join(lines)
568 568
569 569 def check_complete(self, cell: str):
570 570 """Return whether a block of code is ready to execute, or should be continued
571 571
572 572 Parameters
573 573 ----------
574 574 source : string
575 575 Python input code, which can be multiline.
576 576
577 577 Returns
578 578 -------
579 579 status : str
580 580 One of 'complete', 'incomplete', or 'invalid' if source is not a
581 581 prefix of valid code.
582 582 indent_spaces : int or None
583 583 The number of spaces by which to indent the next line of code. If
584 584 status is not 'incomplete', this is None.
585 585 """
586 586 # Remember if the lines ends in a new line.
587 587 ends_with_newline = False
588 588 for character in reversed(cell):
589 589 if character == '\n':
590 590 ends_with_newline = True
591 591 break
592 592 elif character.strip():
593 593 break
594 594 else:
595 595 continue
596 596
597 597 if ends_with_newline:
598 598 # Append an newline for consistent tokenization
599 599 # See https://bugs.python.org/issue33899
600 600 cell += '\n'
601 601
602 602 lines = cell.splitlines(keepends=True)
603 603
604 604 if not lines:
605 605 return 'complete', None
606 606
607 607 if lines[-1].endswith('\\'):
608 608 # Explicit backslash continuation
609 609 return 'incomplete', find_last_indent(lines)
610 610
611 611 try:
612 612 for transform in self.cleanup_transforms:
613 613 lines = transform(lines)
614 614 except SyntaxError:
615 615 return 'invalid', None
616 616
617 617 if lines[0].startswith('%%'):
618 618 # Special case for cell magics - completion marked by blank line
619 619 if lines[-1].strip():
620 620 return 'incomplete', find_last_indent(lines)
621 621 else:
622 622 return 'complete', None
623 623
624 624 try:
625 625 for transform in self.line_transforms:
626 626 lines = transform(lines)
627 627 lines = self.do_token_transforms(lines)
628 628 except SyntaxError:
629 629 return 'invalid', None
630 630
631 631 tokens_by_line = make_tokens_by_line(lines)
632 632
633 633 if not tokens_by_line:
634 634 return 'incomplete', find_last_indent(lines)
635 635
636 636 if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
637 637 # We're in a multiline string or expression
638 638 return 'incomplete', find_last_indent(lines)
639 639
640 640 newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER}
641 641
642 642 # Remove newline_types for the list of tokens
643 643 while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \
644 644 and tokens_by_line[-1][-1].type in newline_types:
645 645 tokens_by_line.pop()
646 646
647
648 647 while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
649 648 tokens_by_line[-1].pop()
650 649
651 650 if len(tokens_by_line) == 1 and not tokens_by_line[-1]:
652 651 return 'incomplete', 0
653 652
654 653 if tokens_by_line[-1][-1].string == ':':
655 654 # The last line starts a block (e.g. 'if foo:')
656 655 ix = 0
657 656 while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
658 657 ix += 1
659 658
660 659 indent = tokens_by_line[-1][ix].start[1]
661 660 return 'incomplete', indent + 4
662 661
663 662 if tokens_by_line[-1][0].line.endswith('\\'):
664 663 return 'incomplete', None
665 664
666 665 # At this point, our checks think the code is complete (or invalid).
667 666 # We'll use codeop.compile_command to check this with the real parser
668 667 try:
669 668 with warnings.catch_warnings():
670 669 warnings.simplefilter('error', SyntaxWarning)
671 670 res = compile_command(''.join(lines), symbol='exec')
672 671 except (SyntaxError, OverflowError, ValueError, TypeError,
673 672 MemoryError, SyntaxWarning):
674 673 return 'invalid', None
675 674 else:
676 675 if res is None:
677 676 return 'incomplete', find_last_indent(lines)
678 677
679 678 if tokens_by_line[-1][-1].type == tokenize.DEDENT:
680 679 if ends_with_newline:
681 680 return 'complete', None
682 681 return 'incomplete', find_last_indent(lines)
683 682
684 if len(tokens_by_line[-1]) <= 1:
685 return 'incomplete', find_last_indent(lines)
686
687 683 # If there's a blank line at the end, assume we're ready to execute
688 684 if not lines[-1].strip():
689 685 return 'complete', None
690 686
691 687 return 'complete', None
692 688
693 689
694 690 def find_last_indent(lines):
695 691 m = _indent_re.match(lines[-1])
696 692 if not m:
697 693 return 0
698 694 return len(m.group(0).replace('\t', ' '*4))
General Comments 0
You need to be logged in to leave comments. Login now