##// END OF EJS Templates
Backport PR #10157: Fix detection of complete input with line continuations inside a multiline string...
Matthias Bussonnier -
Show More
@@ -126,25 +126,18 b' class TokenInputTransformer(InputTransformer):'
126 126 """
127 127 def __init__(self, func):
128 128 self.func = func
129 self.current_line = ""
130 self.line_used = False
129 self.buf = []
131 130 self.reset_tokenizer()
132 131
133 132 def reset_tokenizer(self):
134 self.tokenizer = generate_tokens(self.get_line)
135
136 def get_line(self):
137 if self.line_used:
138 raise TokenError
139 self.line_used = True
140 return self.current_line
133 it = iter(self.buf)
134 self.tokenizer = generate_tokens(it.__next__)
141 135
142 136 def push(self, line):
143 self.current_line += line + "\n"
144 if self.current_line.isspace():
137 self.buf.append(line + '\n')
138 if all(l.isspace() for l in self.buf):
145 139 return self.reset()
146 140
147 self.line_used = False
148 141 tokens = []
149 142 stop_at_NL = False
150 143 try:
@@ -164,13 +157,13 b' class TokenInputTransformer(InputTransformer):'
164 157 return self.output(tokens)
165 158
166 159 def output(self, tokens):
167 self.current_line = ""
160 self.buf.clear()
168 161 self.reset_tokenizer()
169 162 return untokenize(self.func(tokens)).rstrip('\n')
170 163
171 164 def reset(self):
172 l = self.current_line
173 self.current_line = ""
165 l = ''.join(self.buf)
166 self.buf.clear()
174 167 self.reset_tokenizer()
175 168 if l:
176 169 return l.rstrip('\n')
@@ -415,6 +415,11 b' def test_assemble_python_lines():'
415 415 (u"2,", None),
416 416 (None, u"a = [1,\n2,"),
417 417 ],
418 [(u"a = '''", None), # Test line continuation within a multi-line string
419 (u"abc\\", None),
420 (u"def", None),
421 (u"'''", u"a = '''\nabc\\\ndef\n'''"),
422 ],
418 423 ] + syntax_ml['multiline_datastructure']
419 424 for example in tests:
420 425 transform_checker(example, ipt.assemble_python_lines)
General Comments 0
You need to be logged in to leave comments. Login now