Show More
@@ -481,18 +481,6 b' def make_tokens_by_line(lines):' | |||
|
481 | 481 | if not tokens_by_line[-1]: |
|
482 | 482 | tokens_by_line.pop() |
|
483 | 483 | |
|
484 | # Convert if using cpython tokenize | |
|
485 | # upstream bug was fixed in Python 3.7.1, so once we drop 3.7 this can likely be removed. | |
|
486 | if (list(map(lambda x: x.type, tokens_by_line[-1])) == | |
|
487 | [tokenize.DEDENT] * (len(tokens_by_line[-1]) - 1) + [tokenize.ENDMARKER]): | |
|
488 | if ( | |
|
489 | len(tokens_by_line) > 1 and | |
|
490 | len(tokens_by_line[-2]) > 0 and | |
|
491 | tokens_by_line[-2][-1].type == tokenize.NEWLINE | |
|
492 | ): | |
|
493 | tokens_by_line[-2].pop() | |
|
494 | tokens_by_line[-2] += tokens_by_line[-1] | |
|
495 | tokens_by_line.pop() | |
|
496 | 484 | |
|
497 | 485 | return tokens_by_line |
|
498 | 486 | |
@@ -611,7 +599,7 b' class TransformerManager:' | |||
|
611 | 599 | else: |
|
612 | 600 | continue |
|
613 | 601 | |
|
614 | if ends_with_newline: | |
|
602 | if not ends_with_newline: | |
|
615 | 603 | # Append an newline for consistent tokenization |
|
616 | 604 | # See https://bugs.python.org/issue33899 |
|
617 | 605 | cell += '\n' |
@@ -656,10 +644,13 b' class TransformerManager:' | |||
|
656 | 644 | |
|
657 | 645 | newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} |
|
658 | 646 | |
|
659 | # Remove newline_types for the list of tokens | |
|
660 | while len(tokens_by_line) > 1 and len(tokens_by_line[-1]) == 1 \ | |
|
661 |
|
|
|
662 | tokens_by_line.pop() | |
|
647 | # Pop the last line which only contains DEDENTs and ENDMARKER | |
|
648 | last_token_line = None | |
|
649 | if {t.type for t in tokens_by_line[-1]} in [ | |
|
650 | {tokenize.DEDENT, tokenize.ENDMARKER}, | |
|
651 | {tokenize.ENDMARKER} | |
|
652 | ] and len(tokens_by_line) > 1: | |
|
653 | last_token_line = tokens_by_line.pop() | |
|
663 | 654 | |
|
664 | 655 | while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types: |
|
665 | 656 | tokens_by_line[-1].pop() |
@@ -667,15 +658,7 b' class TransformerManager:' | |||
|
667 | 658 | if len(tokens_by_line) == 1 and not tokens_by_line[-1]: |
|
668 | 659 | return 'incomplete', 0 |
|
669 | 660 | |
|
670 | new_block = False | |
|
671 | for token in reversed(tokens_by_line[-1]): | |
|
672 | if token.type == tokenize.DEDENT: | |
|
673 | continue | |
|
674 | elif token.string == ':': | |
|
675 | new_block = True | |
|
676 | break | |
|
677 | ||
|
678 | if new_block: | |
|
661 | if tokens_by_line[-1][-1].string == ':': | |
|
679 | 662 | # The last line starts a block (e.g. 'if foo:') |
|
680 | 663 | ix = 0 |
|
681 | 664 | while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}: |
@@ -700,7 +683,7 b' class TransformerManager:' | |||
|
700 | 683 | if res is None: |
|
701 | 684 | return 'incomplete', find_last_indent(lines) |
|
702 | 685 | |
|
703 |
if |
|
|
686 | if last_token_line and last_token_line[0].type == tokenize.DEDENT: | |
|
704 | 687 | if ends_with_newline: |
|
705 | 688 | return 'complete', None |
|
706 | 689 | return 'incomplete', find_last_indent(lines) |
General Comments 0
You need to be logged in to leave comments.
Login now