From 61fd60ead7a336abf4351944a2d67ab09e6e295d 2013-03-31 09:03:54 From: Thomas Kluyver Date: 2013-03-31 09:03:54 Subject: [PATCH] Revised input transformation framework. --- diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index c27a042..f6401fe 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -82,6 +82,7 @@ from IPython.core.inputtransformer import (leading_indent, escaped_transformer, assign_from_magic, assign_from_system, + assemble_python_lines, ) # Temporary! @@ -499,29 +500,55 @@ class IPythonInputSplitter(InputSplitter): # Flag to track when a transformer has stored input that it hasn't given # back yet. transformer_accumulating = False + + # Flag to track when assemble_python_lines has stored input that it hasn't + # given back yet. + within_python_line = False # Private attributes # List with lines of raw input accumulated so far. _buffer_raw = None - def __init__(self, input_mode=None, transforms=None): + def __init__(self, input_mode=None, physical_line_transforms=None, + logical_line_transforms=None, python_line_transforms=None): super(IPythonInputSplitter, self).__init__(input_mode) self._buffer_raw = [] self._validate = True - if transforms is not None: - self.transforms = transforms - else: - self.transforms = [leading_indent(), - classic_prompt(), - ipy_prompt(), - cellmagic(), - assemble_logical_lines(), - help_end(), - escaped_transformer(), - assign_from_magic(), - assign_from_system(), - ] + + self.physical_line_transforms = physical_line_transforms or \ + [leading_indent(), + classic_prompt(), + ipy_prompt(), + ] + + self.assemble_logical_lines = assemble_logical_lines() + self.logical_line_transforms = logical_line_transforms or \ + [cellmagic(), + help_end(), + escaped_transformer(), + assign_from_magic(), + assign_from_system(), + ] + + self.assemble_python_lines = assemble_python_lines() + self.python_line_transforms = python_line_transforms or [] + + @property + def transforms(self): + "Quick access to all transformers." + return self.physical_line_transforms + \ + [self.assemble_logical_lines] + self.logical_line_transforms + \ + [self.assemble_python_lines] + self.python_line_transforms + + @property + def transforms_in_use(self): + """Transformers, excluding logical line transformers if we're in a + Python line.""" + t = self.physical_line_transforms + [self.assemble_logical_lines] + if not self.within_python_line: + t += self.logical_line_transforms + return t + [self.assemble_python_lines] + self.python_line_transforms def reset(self): """Reset the input buffer and associated state.""" @@ -533,12 +560,18 @@ class IPythonInputSplitter(InputSplitter): t.reset() def flush_transformers(self): + def _flush(transform, out): + if out is not None: + tmp = transform.push(out) + return tmp or transform.reset() or None + else: + return transform.reset() or None + out = None - for t in self.transforms: - tmp = t.reset() - if tmp: - out = tmp - if out: + for t in self.transforms_in_use: + out = _flush(t, out) + + if out is not None: self._store(out) def source_raw_reset(self): @@ -641,11 +674,39 @@ class IPythonInputSplitter(InputSplitter): def push_line(self, line): buf = self._buffer - for transformer in self.transforms: + + def _accumulating(dbg): + #print(dbg) + self.transformer_accumulating = True + return False + + for transformer in self.physical_line_transforms: + line = transformer.push(line) + if line is None: + return _accumulating(transformer) + + line = self.assemble_logical_lines.push(line) + if line is None: + return _accumulating('acc logical line') + + if not self.within_python_line: + for transformer in self.logical_line_transforms: + line = transformer.push(line) + if line is None: + return _accumulating(transformer) + + line = self.assemble_python_lines.push(line) + if line is None: + self.within_python_line = True + return _accumulating('acc python line') + else: + self.within_python_line = False + + for transformer in self.python_line_transforms: line = transformer.push(line) if line is None: - self.transformer_accumulating = True - return False + return _accumulating(transformer) + #print("transformers clear") #debug self.transformer_accumulating = False return super(IPythonInputSplitter, self).push(line) diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index d3bb346..e34f2bb 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -127,7 +127,7 @@ class TokenInputTransformer(InputTransformer): def __init__(self, func): self.func = func self.current_line = "" - self.line_used= False + self.line_used = False self.reset_tokenizer() def reset_tokenizer(self): @@ -141,19 +141,29 @@ class TokenInputTransformer(InputTransformer): def push(self, line): self.current_line += line + "\n" + if self.current_line.isspace(): + return self.reset() + self.line_used = False tokens = [] + stop_at_NL = False try: for intok in self.tokenizer: tokens.append(intok) - if intok[0] in (tokenize.NEWLINE, tokenize.NL): + t = intok[0] + if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL): # Stop before we try to pull a line we don't have yet break + elif t in (tokenize.COMMENT, tokenize.ERRORTOKEN): + stop_at_NL = True except tokenize.TokenError: # Multi-line statement - stop and try again with the next line self.reset_tokenizer() return None + return self.output(tokens) + + def output(self, tokens): self.current_line = "" self.reset_tokenizer() return untokenize(self.func(tokens)).rstrip('\n') @@ -161,12 +171,35 @@ class TokenInputTransformer(InputTransformer): def reset(self): l = self.current_line self.current_line = "" + self.reset_tokenizer() if l: return l.rstrip('\n') -@TokenInputTransformer.wrap -def assemble_logical_lines(tokens): - return tokens +class assemble_python_lines(TokenInputTransformer): + def __init__(self): + super(assemble_python_lines, self).__init__(None) + + def output(self, tokens): + return self.reset() + +@CoroutineInputTransformer.wrap +def assemble_logical_lines(): + """Join lines following explicit line continuations (\)""" + line = '' + while True: + line = (yield line) + if not line or line.isspace(): + continue + + parts = [] + while line is not None: + parts.append(line.rstrip('\\')) + if not line.endswith('\\'): + break + line = (yield None) + + # Output + line = ' '.join(parts) # Utilities def _make_help_call(target, esc, lspace, next_input=None): diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 7bf992a..a351bb0 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -270,6 +270,7 @@ class InputSplitterTestCase(unittest.TestCase): isp = self.isp self.assertFalse(isp.push('if 1:')) for line in [' x=1', '# a comment', ' y=2']: + print(line) self.assertTrue(isp.push(line)) def test_push3(self): diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 5e9df92..aad3abd 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -356,8 +356,8 @@ def test_token_input_transformer(): [ [(u"a = 1.2; b = '''x", None), (u"y'''", u_fmt(u"a =Decimal ({u}'1.2');b ='''x\ny'''")), ], - [(u"a = [1.2,", u_fmt(u"a =[Decimal ({u}'1.2'),")), - (u"3]", u"3 ]"), + [(u"a = [1.2,", None), + (u"3]", u_fmt(u"a =[Decimal ({u}'1.2'),\n3 ]")), ], [(u"a = '''foo", None), # Test resetting when within a multi-line string (u"bar", None),