Show More
@@ -152,7 +152,12 def find_next_indent(code): | |||||
152 | if not tokens: |
|
152 | if not tokens: | |
153 | return 0 |
|
153 | return 0 | |
154 |
|
154 | |||
155 | while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT, tokenize.ERRORTOKEN}): |
|
155 | while tokens[-1].type in { | |
|
156 | tokenize.DEDENT, | |||
|
157 | tokenize.NEWLINE, | |||
|
158 | tokenize.COMMENT, | |||
|
159 | tokenize.ERRORTOKEN, | |||
|
160 | }: | |||
156 | tokens.pop() |
|
161 | tokens.pop() | |
157 |
|
162 | |||
158 | # Starting in Python 3.12, the tokenize module adds implicit newlines at the end |
|
163 | # Starting in Python 3.12, the tokenize module adds implicit newlines at the end |
@@ -292,17 +292,18 class SystemAssign(TokenTransformBase): | |||||
292 | def find_post_312(cls, tokens_by_line): |
|
292 | def find_post_312(cls, tokens_by_line): | |
293 | for line in tokens_by_line: |
|
293 | for line in tokens_by_line: | |
294 | assign_ix = _find_assign_op(line) |
|
294 | assign_ix = _find_assign_op(line) | |
295 | if (assign_ix is not None) \ |
|
295 | if ( | |
296 | and not line[assign_ix].line.strip().startswith('=') \ |
|
296 | (assign_ix is not None) | |
297 | and (len(line) >= assign_ix + 2) \ |
|
297 | and not line[assign_ix].line.strip().startswith("=") | |
298 |
|
|
298 | and (len(line) >= assign_ix + 2) | |
299 |
|
|
299 | and (line[assign_ix + 1].type == tokenize.OP) | |
|
300 | and (line[assign_ix + 1].string == "!") | |||
|
301 | ): | |||
300 | return cls(line[assign_ix + 1].start) |
|
302 | return cls(line[assign_ix + 1].start) | |
301 |
|
303 | |||
302 | @classmethod |
|
304 | @classmethod | |
303 | def find(cls, tokens_by_line): |
|
305 | def find(cls, tokens_by_line): | |
304 | """Find the first system assignment (a = !foo) in the cell. |
|
306 | """Find the first system assignment (a = !foo) in the cell.""" | |
305 | """ |
|
|||
306 | if sys.version_info < (3, 12): |
|
307 | if sys.version_info < (3, 12): | |
307 | return cls.find_pre_312(tokens_by_line) |
|
308 | return cls.find_pre_312(tokens_by_line) | |
308 | return cls.find_post_312(tokens_by_line) |
|
309 | return cls.find_post_312(tokens_by_line) | |
@@ -531,8 +532,9 def make_tokens_by_line(lines:List[str]): | |||||
531 | ) |
|
532 | ) | |
532 | parenlev = 0 |
|
533 | parenlev = 0 | |
533 | try: |
|
534 | try: | |
534 |
for token in tokenutil.generate_tokens_catch_errors( |
|
535 | for token in tokenutil.generate_tokens_catch_errors( | |
535 | extra_errors_to_catch=['expected EOF']): |
|
536 | iter(lines).__next__, extra_errors_to_catch=["expected EOF"] | |
|
537 | ): | |||
536 | tokens_by_line[-1].append(token) |
|
538 | tokens_by_line[-1].append(token) | |
537 | if (token.type == NEWLINE) \ |
|
539 | if (token.type == NEWLINE) \ | |
538 | or ((token.type == NL) and (parenlev <= 0)): |
|
540 | or ((token.type == NL) and (parenlev <= 0)): | |
@@ -701,8 +703,8 class TransformerManager: | |||||
701 | for line in reversed(lines): |
|
703 | for line in reversed(lines): | |
702 | if not line.strip(): |
|
704 | if not line.strip(): | |
703 | continue |
|
705 | continue | |
704 |
elif line.strip( |
|
706 | elif line.strip("\n").endswith("\\"): | |
705 |
return |
|
707 | return "incomplete", find_last_indent(lines) | |
706 | else: |
|
708 | else: | |
707 | break |
|
709 | break | |
708 |
|
710 | |||
@@ -742,8 +744,10 class TransformerManager: | |||||
742 | if not tokens_by_line: |
|
744 | if not tokens_by_line: | |
743 | return 'incomplete', find_last_indent(lines) |
|
745 | return 'incomplete', find_last_indent(lines) | |
744 |
|
746 | |||
745 | if (tokens_by_line[-1][-1].type != tokenize.ENDMARKER |
|
747 | if ( | |
746 |
|
|
748 | tokens_by_line[-1][-1].type != tokenize.ENDMARKER | |
|
749 | and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN | |||
|
750 | ): | |||
747 | # We're in a multiline string or expression |
|
751 | # We're in a multiline string or expression | |
748 | return 'incomplete', find_last_indent(lines) |
|
752 | return 'incomplete', find_last_indent(lines) | |
749 |
|
753 |
@@ -297,6 +297,7 def test_find_assign_op_dedent(): | |||||
297 | _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6 |
|
297 | _find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6 | |
298 | ) |
|
298 | ) | |
299 |
|
299 | |||
|
300 | ||||
300 | extra_closing_paren_param = ( |
|
301 | extra_closing_paren_param = ( | |
301 | pytest.param("(\n))", "invalid", None) |
|
302 | pytest.param("(\n))", "invalid", None) | |
302 | if sys.version_info >= (3, 12) |
|
303 | if sys.version_info >= (3, 12) |
@@ -21,9 +21,13 def generate_tokens(readline): | |||||
21 | # catch EOF error |
|
21 | # catch EOF error | |
22 | return |
|
22 | return | |
23 |
|
23 | |||
|
24 | ||||
24 | def generate_tokens_catch_errors(readline, extra_errors_to_catch=None): |
|
25 | def generate_tokens_catch_errors(readline, extra_errors_to_catch=None): | |
25 | default_errors_to_catch = ['unterminated string literal', 'invalid non-printable character', |
|
26 | default_errors_to_catch = [ | |
26 | 'after line continuation character'] |
|
27 | "unterminated string literal", | |
|
28 | "invalid non-printable character", | |||
|
29 | "after line continuation character", | |||
|
30 | ] | |||
27 | assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list) |
|
31 | assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list) | |
28 | errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or []) |
|
32 | errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or []) | |
29 |
|
33 | |||
@@ -40,12 +44,13 def generate_tokens_catch_errors(readline, extra_errors_to_catch=None): | |||||
40 | line = tokens[-1].line |
|
44 | line = tokens[-1].line | |
41 | else: |
|
45 | else: | |
42 | start = end = (1, 0) |
|
46 | start = end = (1, 0) | |
43 |
line = |
|
47 | line = "" | |
44 |
yield tokenize.TokenInfo(tokenize.ERRORTOKEN, |
|
48 | yield tokenize.TokenInfo(tokenize.ERRORTOKEN, "", start, end, line) | |
45 | else: |
|
49 | else: | |
46 | # Catch EOF |
|
50 | # Catch EOF | |
47 | raise |
|
51 | raise | |
48 |
|
52 | |||
|
53 | ||||
49 | def line_at_cursor(cell, cursor_pos=0): |
|
54 | def line_at_cursor(cell, cursor_pos=0): | |
50 | """Return the line in a cell at a given cursor position |
|
55 | """Return the line in a cell at a given cursor position | |
51 |
|
56 |
General Comments 0
You need to be logged in to leave comments.
Login now