Show More
@@ -82,6 +82,7 b' from IPython.core.inputtransformer import (leading_indent,' | |||
|
82 | 82 | escaped_transformer, |
|
83 | 83 | assign_from_magic, |
|
84 | 84 | assign_from_system, |
|
85 | assemble_python_lines, | |
|
85 | 86 | ) |
|
86 | 87 | |
|
87 | 88 | # Temporary! |
@@ -499,29 +500,55 b' class IPythonInputSplitter(InputSplitter):' | |||
|
499 | 500 | # Flag to track when a transformer has stored input that it hasn't given |
|
500 | 501 | # back yet. |
|
501 | 502 | transformer_accumulating = False |
|
503 | ||
|
504 | # Flag to track when assemble_python_lines has stored input that it hasn't | |
|
505 | # given back yet. | |
|
506 | within_python_line = False | |
|
502 | 507 | |
|
503 | 508 | # Private attributes |
|
504 | 509 | |
|
505 | 510 | # List with lines of raw input accumulated so far. |
|
506 | 511 | _buffer_raw = None |
|
507 | 512 | |
|
508 |
def __init__(self, input_mode=None, transforms=None |
|
|
513 | def __init__(self, input_mode=None, physical_line_transforms=None, | |
|
514 | logical_line_transforms=None, python_line_transforms=None): | |
|
509 | 515 | super(IPythonInputSplitter, self).__init__(input_mode) |
|
510 | 516 | self._buffer_raw = [] |
|
511 | 517 | self._validate = True |
|
512 | if transforms is not None: | |
|
513 |
|
|
|
514 | else: | |
|
515 | self.transforms = [leading_indent(), | |
|
516 |
|
|
|
517 | ipy_prompt(), | |
|
518 | cellmagic(), | |
|
519 | assemble_logical_lines(), | |
|
520 | help_end(), | |
|
521 | escaped_transformer(), | |
|
522 | assign_from_magic(), | |
|
523 | assign_from_system(), | |
|
524 | ] | |
|
518 | ||
|
519 | self.physical_line_transforms = physical_line_transforms or \ | |
|
520 | [leading_indent(), | |
|
521 | classic_prompt(), | |
|
522 | ipy_prompt(), | |
|
523 | ] | |
|
524 | ||
|
525 | self.assemble_logical_lines = assemble_logical_lines() | |
|
526 | self.logical_line_transforms = logical_line_transforms or \ | |
|
527 | [cellmagic(), | |
|
528 | help_end(), | |
|
529 | escaped_transformer(), | |
|
530 | assign_from_magic(), | |
|
531 | assign_from_system(), | |
|
532 | ] | |
|
533 | ||
|
534 | self.assemble_python_lines = assemble_python_lines() | |
|
535 | self.python_line_transforms = python_line_transforms or [] | |
|
536 | ||
|
537 | @property | |
|
538 | def transforms(self): | |
|
539 | "Quick access to all transformers." | |
|
540 | return self.physical_line_transforms + \ | |
|
541 | [self.assemble_logical_lines] + self.logical_line_transforms + \ | |
|
542 | [self.assemble_python_lines] + self.python_line_transforms | |
|
543 | ||
|
544 | @property | |
|
545 | def transforms_in_use(self): | |
|
546 | """Transformers, excluding logical line transformers if we're in a | |
|
547 | Python line.""" | |
|
548 | t = self.physical_line_transforms + [self.assemble_logical_lines] | |
|
549 | if not self.within_python_line: | |
|
550 | t += self.logical_line_transforms | |
|
551 | return t + [self.assemble_python_lines] + self.python_line_transforms | |
|
525 | 552 | |
|
526 | 553 | def reset(self): |
|
527 | 554 | """Reset the input buffer and associated state.""" |
@@ -533,12 +560,18 b' class IPythonInputSplitter(InputSplitter):' | |||
|
533 | 560 | t.reset() |
|
534 | 561 | |
|
535 | 562 | def flush_transformers(self): |
|
563 | def _flush(transform, out): | |
|
564 | if out is not None: | |
|
565 | tmp = transform.push(out) | |
|
566 | return tmp or transform.reset() or None | |
|
567 | else: | |
|
568 | return transform.reset() or None | |
|
569 | ||
|
536 | 570 | out = None |
|
537 | for t in self.transforms: | |
|
538 |
t |
|
|
539 | if tmp: | |
|
540 | out = tmp | |
|
541 | if out: | |
|
571 | for t in self.transforms_in_use: | |
|
572 | out = _flush(t, out) | |
|
573 | ||
|
574 | if out is not None: | |
|
542 | 575 | self._store(out) |
|
543 | 576 | |
|
544 | 577 | def source_raw_reset(self): |
@@ -641,11 +674,39 b' class IPythonInputSplitter(InputSplitter):' | |||
|
641 | 674 | |
|
642 | 675 | def push_line(self, line): |
|
643 | 676 | buf = self._buffer |
|
644 | for transformer in self.transforms: | |
|
677 | ||
|
678 | def _accumulating(dbg): | |
|
679 | #print(dbg) | |
|
680 | self.transformer_accumulating = True | |
|
681 | return False | |
|
682 | ||
|
683 | for transformer in self.physical_line_transforms: | |
|
684 | line = transformer.push(line) | |
|
685 | if line is None: | |
|
686 | return _accumulating(transformer) | |
|
687 | ||
|
688 | line = self.assemble_logical_lines.push(line) | |
|
689 | if line is None: | |
|
690 | return _accumulating('acc logical line') | |
|
691 | ||
|
692 | if not self.within_python_line: | |
|
693 | for transformer in self.logical_line_transforms: | |
|
694 | line = transformer.push(line) | |
|
695 | if line is None: | |
|
696 | return _accumulating(transformer) | |
|
697 | ||
|
698 | line = self.assemble_python_lines.push(line) | |
|
699 | if line is None: | |
|
700 | self.within_python_line = True | |
|
701 | return _accumulating('acc python line') | |
|
702 | else: | |
|
703 | self.within_python_line = False | |
|
704 | ||
|
705 | for transformer in self.python_line_transforms: | |
|
645 | 706 | line = transformer.push(line) |
|
646 | 707 | if line is None: |
|
647 |
|
|
|
648 | return False | |
|
708 | return _accumulating(transformer) | |
|
649 | 709 | |
|
710 | #print("transformers clear") #debug | |
|
650 | 711 | self.transformer_accumulating = False |
|
651 | 712 | return super(IPythonInputSplitter, self).push(line) |
@@ -127,7 +127,7 b' class TokenInputTransformer(InputTransformer):' | |||
|
127 | 127 | def __init__(self, func): |
|
128 | 128 | self.func = func |
|
129 | 129 | self.current_line = "" |
|
130 | self.line_used= False | |
|
130 | self.line_used = False | |
|
131 | 131 | self.reset_tokenizer() |
|
132 | 132 | |
|
133 | 133 | def reset_tokenizer(self): |
@@ -141,19 +141,29 b' class TokenInputTransformer(InputTransformer):' | |||
|
141 | 141 | |
|
142 | 142 | def push(self, line): |
|
143 | 143 | self.current_line += line + "\n" |
|
144 | if self.current_line.isspace(): | |
|
145 | return self.reset() | |
|
146 | ||
|
144 | 147 | self.line_used = False |
|
145 | 148 | tokens = [] |
|
149 | stop_at_NL = False | |
|
146 | 150 | try: |
|
147 | 151 | for intok in self.tokenizer: |
|
148 | 152 | tokens.append(intok) |
|
149 | if intok[0] in (tokenize.NEWLINE, tokenize.NL): | |
|
153 | t = intok[0] | |
|
154 | if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL): | |
|
150 | 155 | # Stop before we try to pull a line we don't have yet |
|
151 | 156 | break |
|
157 | elif t in (tokenize.COMMENT, tokenize.ERRORTOKEN): | |
|
158 | stop_at_NL = True | |
|
152 | 159 | except tokenize.TokenError: |
|
153 | 160 | # Multi-line statement - stop and try again with the next line |
|
154 | 161 | self.reset_tokenizer() |
|
155 | 162 | return None |
|
156 | 163 | |
|
164 | return self.output(tokens) | |
|
165 | ||
|
166 | def output(self, tokens): | |
|
157 | 167 | self.current_line = "" |
|
158 | 168 | self.reset_tokenizer() |
|
159 | 169 | return untokenize(self.func(tokens)).rstrip('\n') |
@@ -161,12 +171,35 b' class TokenInputTransformer(InputTransformer):' | |||
|
161 | 171 | def reset(self): |
|
162 | 172 | l = self.current_line |
|
163 | 173 | self.current_line = "" |
|
174 | self.reset_tokenizer() | |
|
164 | 175 | if l: |
|
165 | 176 | return l.rstrip('\n') |
|
166 | 177 | |
|
167 | @TokenInputTransformer.wrap | |
|
168 | def assemble_logical_lines(tokens): | |
|
169 | return tokens | |
|
178 | class assemble_python_lines(TokenInputTransformer): | |
|
179 | def __init__(self): | |
|
180 | super(assemble_python_lines, self).__init__(None) | |
|
181 | ||
|
182 | def output(self, tokens): | |
|
183 | return self.reset() | |
|
184 | ||
|
185 | @CoroutineInputTransformer.wrap | |
|
186 | def assemble_logical_lines(): | |
|
187 | """Join lines following explicit line continuations (\)""" | |
|
188 | line = '' | |
|
189 | while True: | |
|
190 | line = (yield line) | |
|
191 | if not line or line.isspace(): | |
|
192 | continue | |
|
193 | ||
|
194 | parts = [] | |
|
195 | while line is not None: | |
|
196 | parts.append(line.rstrip('\\')) | |
|
197 | if not line.endswith('\\'): | |
|
198 | break | |
|
199 | line = (yield None) | |
|
200 | ||
|
201 | # Output | |
|
202 | line = ' '.join(parts) | |
|
170 | 203 | |
|
171 | 204 | # Utilities |
|
172 | 205 | def _make_help_call(target, esc, lspace, next_input=None): |
@@ -270,6 +270,7 b' class InputSplitterTestCase(unittest.TestCase):' | |||
|
270 | 270 | isp = self.isp |
|
271 | 271 | self.assertFalse(isp.push('if 1:')) |
|
272 | 272 | for line in [' x=1', '# a comment', ' y=2']: |
|
273 | print(line) | |
|
273 | 274 | self.assertTrue(isp.push(line)) |
|
274 | 275 | |
|
275 | 276 | def test_push3(self): |
@@ -356,8 +356,8 b' def test_token_input_transformer():' | |||
|
356 | 356 | [ [(u"a = 1.2; b = '''x", None), |
|
357 | 357 | (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")), |
|
358 | 358 | ], |
|
359 | [(u"a = [1.2,", u_fmt(u"a =[Decimal ({u}'1.2'),")), | |
|
360 | (u"3]", u"3 ]"), | |
|
359 | [(u"a = [1.2,", None), | |
|
360 | (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")), | |
|
361 | 361 | ], |
|
362 | 362 | [(u"a = '''foo", None), # Test resetting when within a multi-line string |
|
363 | 363 | (u"bar", None), |
General Comments 0
You need to be logged in to leave comments.
Login now