diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py index eb1f8fb..417b436 100644 --- a/IPython/core/inputsplitter.py +++ b/IPython/core/inputsplitter.py @@ -508,14 +508,14 @@ class IPythonInputSplitter(InputSplitter): super(IPythonInputSplitter, self).__init__(input_mode) self._buffer_raw = [] self._validate = True - self.transforms = [leading_indent, - classic_prompt, - ipy_prompt, - cellmagic, - help_end, - escaped_transformer, - assign_from_magic, - assign_from_system, + self.transforms = [leading_indent(), + classic_prompt(), + ipy_prompt(), + cellmagic(), + help_end(), + escaped_transformer(), + assign_from_magic(), + assign_from_system(), ] def reset(self): diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 5beccf4..314deb4 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -44,29 +44,34 @@ class InputTransformer(object): look_in_string = False -class StatelessInputTransformer(InputTransformer): - """Decorator for a stateless input transformer implemented as a function.""" - def __init__(self, func): - self.func = func - - def push(self, line): - return self.func(line) - - def reset(self): - pass - -class CoroutineInputTransformer(InputTransformer): - """Decorator for an input transformer implemented as a coroutine.""" - def __init__(self, coro): - # Prime it - self.coro = coro() - next(self.coro) +def stateless_input_transformer(func): + class StatelessInputTransformer(InputTransformer): + """Decorator for a stateless input transformer implemented as a function.""" + def __init__(self): + self.func = func + + def push(self, line): + return self.func(line) + + def reset(self): + pass - def push(self, line): - return self.coro.send(line) + return StatelessInputTransformer + +def coroutine_input_transformer(coro): + class CoroutineInputTransformer(InputTransformer): + def __init__(self): + # Prime it + self.coro = coro() + next(self.coro) + + def push(self, line): + return self.coro.send(line) + + def reset(self): + return self.coro.send(None) - def reset(self): - return self.coro.send(None) + return CoroutineInputTransformer # Utilities @@ -83,7 +88,7 @@ def _make_help_call(target, esc, lspace, next_input=None): return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \ (lspace, next_input, arg) -@CoroutineInputTransformer +@coroutine_input_transformer def escaped_transformer(): """Translate lines beginning with one of IPython's escape characters.""" @@ -188,7 +193,7 @@ def has_comment(src): pass return(tokenize.COMMENT in toktypes) -@StatelessInputTransformer +@stateless_input_transformer def help_end(line): """Translate lines with ?/?? at the end""" m = _help_end_re.search(line) @@ -204,15 +209,20 @@ def help_end(line): return _make_help_call(target, esc, lspace, next_input) -@CoroutineInputTransformer +@coroutine_input_transformer def cellmagic(): tpl = 'get_ipython().run_cell_magic(%r, %r, %r)' + cellmagic_help_re = re.compile('%%\w+\?') line = '' while True: line = (yield line) if (not line) or (not line.startswith(ESC_MAGIC2)): continue + if cellmagic_help_re.match(line): + # This case will be handled by help_end + continue + first = line body = [] line = (yield None) @@ -223,7 +233,7 @@ def cellmagic(): # Output magic_name, _, first = first.partition(' ') magic_name = magic_name.lstrip(ESC_MAGIC2) - line = tpl % (magic_name, first, '\n'.join(body)) + line = tpl % (magic_name, first, u'\n'.join(body)) def _strip_prompts(prompt1_re, prompt2_re): """Remove matching input prompts from a block of input.""" @@ -246,7 +256,7 @@ def _strip_prompts(prompt1_re, prompt2_re): while line is not None: line = (yield line) -@CoroutineInputTransformer +@coroutine_input_transformer def classic_prompt(): prompt1_re = re.compile(r'^(>>> )') prompt2_re = re.compile(r'^(>>> |^\.\.\. )') @@ -254,7 +264,7 @@ def classic_prompt(): classic_prompt.look_in_string = True -@CoroutineInputTransformer +@coroutine_input_transformer def ipy_prompt(): prompt1_re = re.compile(r'^In \[\d+\]: ') prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )') @@ -262,7 +272,7 @@ def ipy_prompt(): ipy_prompt.look_in_string = True -@CoroutineInputTransformer +@coroutine_input_transformer def leading_indent(): space_re = re.compile(r'^[ \t]+') line = '' @@ -308,14 +318,14 @@ def _special_assignment(assignment_re, template): whole = assignment_re.match(' '.join(parts)) line = template % (whole.group('lhs'), whole.group('cmd')) -@CoroutineInputTransformer +@coroutine_input_transformer def assign_from_system(): assignment_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' r'\s*=\s*!\s*(?P.*)') template = '%s = get_ipython().getoutput(%r)' return _special_assignment(assignment_re, template) -@CoroutineInputTransformer +@coroutine_input_transformer def assign_from_magic(): assignment_re = re.compile(r'(?P(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))' r'\s*=\s*%\s*(?P.*)') diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py index 0d3f195..7bf992a 100644 --- a/IPython/core/tests/test_inputsplitter.py +++ b/IPython/core/tests/test_inputsplitter.py @@ -109,18 +109,6 @@ def test_remove_comments(): ] tt.check_pairs(isp.remove_comments, tests) -def test_has_comment(): - tests = [('text', False), - ('text #comment', True), - ('text #comment\n', True), - ('#comment', True), - ('#comment\n', True), - ('a = "#string"', False), - ('a = "#string" # comment', True), - ('a #comment not "string"', True), - ] - tt.check_pairs(isp.has_comment, tests) - def test_get_input_encoding(): encoding = isp.get_input_encoding() @@ -461,13 +449,16 @@ class IPythonInputTestCase(InputSplitterTestCase): def test_syntax_multiline(self): isp = self.isp for example in syntax_ml.itervalues(): - out_t_parts = [] - raw_parts = [] for line_pairs in example: + out_t_parts = [] + raw_parts = [] for lraw, out_t_part in line_pairs: - isp.push(lraw) - out_t_parts.append(out_t_part) - raw_parts.append(lraw) + if out_t_part is not None: + out_t_parts.append(out_t_part) + + if lraw is not None: + isp.push(lraw) + raw_parts.append(lraw) out, out_raw = isp.source_raw_reset() out_t = '\n'.join(out_t_parts).rstrip() @@ -490,12 +481,10 @@ class BlockIPythonInputTestCase(IPythonInputTestCase): raw_parts = [] out_t_parts = [] for line_pairs in example: - for raw, out_t_part in line_pairs: - raw_parts.append(raw) - out_t_parts.append(out_t_part) + raw_parts, out_t_parts = zip(*line_pairs) - raw = '\n'.join(raw_parts) - out_t = '\n'.join(out_t_parts) + raw = '\n'.join(r for r in raw_parts if r is not None) + out_t = '\n'.join(o for o in out_t_parts if o is not None) isp.push(raw) out, out_raw = isp.source_raw_reset() @@ -509,8 +498,8 @@ class BlockIPythonInputTestCase(IPythonInputTestCase): out_t_parts = [] for line_pairs in example: - raw = '\n'.join(r for r, _ in line_pairs) - out_t = '\n'.join(t for _,t in line_pairs) + raw = '\n'.join(r for r, _ in line_pairs if r is not None) + out_t = '\n'.join(t for _,t in line_pairs if t is not None) out = isp.transform_cell(raw) # Match ignoring trailing whitespace self.assertEqual(out.rstrip(), out_t.rstrip()) @@ -595,9 +584,8 @@ class CellMagicsCommon(object): src = "%%cellm line\nbody\n" sp = self.sp sp.push(src) - nt.assert_equal(sp.cell_magic_parts, ['body\n']) - out = sp.source - ref = u"get_ipython()._run_cached_cell_magic({u}'cellm', {u}'line')\n" + out = sp.source_reset() + ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n" nt.assert_equal(out, py3compat.u_format(ref)) def tearDown(self): diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 307114e..5839ce7 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -8,6 +8,7 @@ u_fmt = py3compat.u_format from IPython.core import inputtransformer as ipt def transform_and_reset(transformer): + transformer = transformer() def transform(inp): try: return transformer.push(inp) @@ -19,6 +20,7 @@ def transform_and_reset(transformer): # Transformer tests def transform_checker(tests, transformer): """Utility to loop over test inputs""" + transformer = transformer() try: for inp, tr in tests: nt.assert_equal(transformer.push(inp), tr) @@ -191,7 +193,8 @@ syntax_ml = \ leading_indent = [ [(' print "hi"','print "hi"'), - (' for a in range(5):','for a in range(5):'), + ], + [(' for a in range(5):','for a in range(5):'), (' a*2',' a*2'), ], [(' a="""','a="""'), @@ -203,12 +206,12 @@ syntax_ml = \ ], cellmagic = - [ [('%%foo a', None), - (None, "get_ipython().run_cell_magic('foo', 'a', '')"), + [ [(u'%%foo a', None), + (None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")), ], - [('%%bar 123', None), - ('hello', None), - ('', "get_ipython().run_cell_magic('bar', '123', 'hello')"), + [(u'%%bar 123', None), + (u'hello', None), + (u'', u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")), ], ], @@ -223,22 +226,22 @@ syntax_ml = \ ], assign_magic = - [ [('a = %bc de \\', None), - ('fg', "a = get_ipython().magic('bc de fg')"), + [ [(u'a = %bc de \\', None), + (u'fg', u_fmt("a = get_ipython().magic({u}'bc de fg')")), ], - [('a = %bc de \\', None), - ('fg\\', None), - (None, "a = get_ipython().magic('bc de fg')"), + [(u'a = %bc de \\', None), + (u'fg\\', None), + (None, u_fmt("a = get_ipython().magic({u}'bc de fg')")), ], ], assign_system = - [ [('a = !bc de \\', None), - ('fg', "a = get_ipython().getoutput('bc de fg')"), + [ [(u'a = !bc de \\', None), + (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), ], - [('a = !bc de \\', None), - ('fg\\', None), - (None, "a = get_ipython().getoutput('bc de fg')"), + [(u'a = !bc de \\', None), + (u'fg\\', None), + (None, u_fmt("a = get_ipython().getoutput({u}'bc de fg')")), ], ], ) @@ -259,6 +262,8 @@ def test_classic_prompt(): tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt']) for example in syntax_ml['classic_prompt']: transform_checker(example, ipt.classic_prompt) + for example in syntax_ml['multiline_datastructure']: + transform_checker(example, ipt.classic_prompt) def test_ipy_prompt(): @@ -303,3 +308,15 @@ def test_escaped_multiline(): def test_cellmagic(): for example in syntax_ml['cellmagic']: transform_checker(example, ipt.cellmagic) + +def test_has_comment(): + tests = [('text', False), + ('text #comment', True), + ('text #comment\n', True), + ('#comment', True), + ('#comment\n', True), + ('a = "#string"', False), + ('a = "#string" # comment', True), + ('a #comment not "string"', True), + ] + tt.check_pairs(ipt.has_comment, tests)