Show More
@@ -126,25 +126,18 b' class TokenInputTransformer(InputTransformer):' | |||||
126 | """ |
|
126 | """ | |
127 | def __init__(self, func): |
|
127 | def __init__(self, func): | |
128 | self.func = func |
|
128 | self.func = func | |
129 |
self. |
|
129 | self.buf = [] | |
130 | self.line_used = False |
|
|||
131 | self.reset_tokenizer() |
|
130 | self.reset_tokenizer() | |
132 |
|
131 | |||
133 | def reset_tokenizer(self): |
|
132 | def reset_tokenizer(self): | |
134 | self.tokenizer = generate_tokens(self.get_line) |
|
133 | it = iter(self.buf) | |
135 |
|
134 | self.tokenizer = generate_tokens(it.__next__) | ||
136 | def get_line(self): |
|
|||
137 | if self.line_used: |
|
|||
138 | raise TokenError |
|
|||
139 | self.line_used = True |
|
|||
140 | return self.current_line |
|
|||
141 |
|
135 | |||
142 | def push(self, line): |
|
136 | def push(self, line): | |
143 |
self. |
|
137 | self.buf.append(line + '\n') | |
144 | if self.current_line.isspace(): |
|
138 | if all(l.isspace() for l in self.buf): | |
145 | return self.reset() |
|
139 | return self.reset() | |
146 |
|
140 | |||
147 | self.line_used = False |
|
|||
148 | tokens = [] |
|
141 | tokens = [] | |
149 | stop_at_NL = False |
|
142 | stop_at_NL = False | |
150 | try: |
|
143 | try: | |
@@ -164,13 +157,13 b' class TokenInputTransformer(InputTransformer):' | |||||
164 | return self.output(tokens) |
|
157 | return self.output(tokens) | |
165 |
|
158 | |||
166 | def output(self, tokens): |
|
159 | def output(self, tokens): | |
167 | self.current_line = "" |
|
160 | self.buf.clear() | |
168 | self.reset_tokenizer() |
|
161 | self.reset_tokenizer() | |
169 | return untokenize(self.func(tokens)).rstrip('\n') |
|
162 | return untokenize(self.func(tokens)).rstrip('\n') | |
170 |
|
163 | |||
171 | def reset(self): |
|
164 | def reset(self): | |
172 |
l = self. |
|
165 | l = ''.join(self.buf) | |
173 | self.current_line = "" |
|
166 | self.buf.clear() | |
174 | self.reset_tokenizer() |
|
167 | self.reset_tokenizer() | |
175 | if l: |
|
168 | if l: | |
176 | return l.rstrip('\n') |
|
169 | return l.rstrip('\n') |
@@ -415,6 +415,11 b' def test_assemble_python_lines():' | |||||
415 | (u"2,", None), |
|
415 | (u"2,", None), | |
416 | (None, u"a = [1,\n2,"), |
|
416 | (None, u"a = [1,\n2,"), | |
417 | ], |
|
417 | ], | |
|
418 | [(u"a = '''", None), # Test line continuation within a multi-line string | |||
|
419 | (u"abc\\", None), | |||
|
420 | (u"def", None), | |||
|
421 | (u"'''", u"a = '''\nabc\\\ndef\n'''"), | |||
|
422 | ], | |||
418 | ] + syntax_ml['multiline_datastructure'] |
|
423 | ] + syntax_ml['multiline_datastructure'] | |
419 | for example in tests: |
|
424 | for example in tests: | |
420 | transform_checker(example, ipt.assemble_python_lines) |
|
425 | transform_checker(example, ipt.assemble_python_lines) |
General Comments 0
You need to be logged in to leave comments.
Login now