Fix tests in IPython.core

IPython/core/inputsplitter.py

@@ -508,14 +508,14 b' class IPythonInputSplitter(InputSplitter):'
         super(IPythonInputSplitter, self).__init__(input_mode)
         self._buffer_raw = []
         self._validate = True
-        self.transforms = [leading_indent,
-                           classic_prompt,
-                           ipy_prompt,
-                           cellmagic,
-                           help_end,
-                           escaped_transformer,
-                           assign_from_magic,
-                           assign_from_system,
+        self.transforms = [leading_indent(),
+                           classic_prompt(),
+                           ipy_prompt(),
+                           cellmagic(),
+                           help_end(),
+                           escaped_transformer(),
+                           assign_from_magic(),
+                           assign_from_system(),
                           ]
 
     def reset(self):

IPython/core/inputtransformer.py

@@ -44,29 +44,34 b' class InputTransformer(object):'
     
     look_in_string = False
 
-class StatelessInputTransformer(InputTransformer):
-    """Decorator for a stateless input transformer implemented as a function."""
-    def __init__(self, func):
-        self.func = func
-    
-    def push(self, line):
-        return self.func(line)
-    
-    def reset(self):
-        pass
-
-class CoroutineInputTransformer(InputTransformer):
-    """Decorator for an input transformer implemented as a coroutine."""
-    def __init__(self, coro):
-        # Prime it
-        self.coro = coro()
-        next(self.coro)
+def stateless_input_transformer(func):
+    class StatelessInputTransformer(InputTransformer):
+        """Decorator for a stateless input transformer implemented as a function."""
+        def __init__(self):
+            self.func = func
+        
+        def push(self, line):
+            return self.func(line)
+        
+        def reset(self):
+            pass
     
-    def push(self, line):
-        return self.coro.send(line)
+    return StatelessInputTransformer
+
+def coroutine_input_transformer(coro):
+    class CoroutineInputTransformer(InputTransformer):
+        def __init__(self):
+            # Prime it
+            self.coro = coro()
+            next(self.coro)
+        
+        def push(self, line):
+            return self.coro.send(line)
+        
+        def reset(self):
+            return self.coro.send(None)
     
-    def reset(self):
-        return self.coro.send(None)
+    return CoroutineInputTransformer
 
 
 # Utilities
@@ -83,7 +88,7 b' def _make_help_call(target, esc, lspace, next_input=None):'
         return '%sget_ipython().set_next_input(%r);get_ipython().magic(%r)' % \
            (lspace, next_input, arg)
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def escaped_transformer():
     """Translate lines beginning with one of IPython's escape characters."""
     
@@ -188,7 +193,7 b' def has_comment(src):'
         pass
     return(tokenize.COMMENT in toktypes)
 
-@StatelessInputTransformer
+@stateless_input_transformer
 def help_end(line):
     """Translate lines with ?/?? at the end"""
     m = _help_end_re.search(line)
@@ -204,15 +209,20 b' def help_end(line):'
     return _make_help_call(target, esc, lspace, next_input)
     
                 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def cellmagic():
     tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
+    cellmagic_help_re = re.compile('%%\w+\?')
     line = ''
     while True:
         line = (yield line)
         if (not line) or (not line.startswith(ESC_MAGIC2)):
             continue
         
+        if cellmagic_help_re.match(line):
+            # This case will be handled by help_end
+            continue
+        
         first = line
         body = []
         line = (yield None)
@@ -223,7 +233,7 b' def cellmagic():'
         # Output
         magic_name, _, first = first.partition(' ')
         magic_name = magic_name.lstrip(ESC_MAGIC2)
-        line = tpl % (magic_name, first, '\n'.join(body))
+        line = tpl % (magic_name, first, u'\n'.join(body))
 
 def _strip_prompts(prompt1_re, prompt2_re):
     """Remove matching input prompts from a block of input."""
@@ -246,7 +256,7 b' def _strip_prompts(prompt1_re, prompt2_re):'
             while line is not None:
                 line = (yield line)
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def classic_prompt():
     prompt1_re = re.compile(r'^(>>> )')
     prompt2_re = re.compile(r'^(>>> |^\.\.\. )')
@@ -254,7 +264,7 b' def classic_prompt():'
 
 classic_prompt.look_in_string = True
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def ipy_prompt():
     prompt1_re = re.compile(r'^In \[\d+\]: ')
     prompt2_re = re.compile(r'^(In \[\d+\]: |^\ \ \ \.\.\.+: )')
@@ -262,7 +272,7 b' def ipy_prompt():'
 
 ipy_prompt.look_in_string = True
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def leading_indent():
     space_re = re.compile(r'^[ \t]+')
     line = ''
@@ -308,14 +318,14 b' def _special_assignment(assignment_re, template):'
         whole = assignment_re.match(' '.join(parts))
         line = template % (whole.group('lhs'), whole.group('cmd'))
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def assign_from_system():
     assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*!\s*(?P<cmd>.*)')
     template = '%s = get_ipython().getoutput(%r)'
     return _special_assignment(assignment_re, template)
 
-@CoroutineInputTransformer
+@coroutine_input_transformer
 def assign_from_magic():
     assignment_re = re.compile(r'(?P<lhs>(\s*)([\w\.]+)((\s*,\s*[\w\.]+)*))'
                                r'\s*=\s*%\s*(?P<cmd>.*)')

IPython/core/tests/test_inputsplitter.py

@@ -109,18 +109,6 b' def test_remove_comments():'
              ]
     tt.check_pairs(isp.remove_comments, tests)
 
-def test_has_comment():
-    tests = [('text', False),
-             ('text #comment', True),
-             ('text #comment\n', True),
-             ('#comment', True),
-             ('#comment\n', True),
-             ('a = "#string"', False),
-             ('a = "#string" # comment', True),
-             ('a #comment not "string"', True),
-             ]
-    tt.check_pairs(isp.has_comment, tests)
-
 
 def test_get_input_encoding():
     encoding = isp.get_input_encoding()
@@ -461,13 +449,16 b' class IPythonInputTestCase(InputSplitterTestCase):'
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
-            out_t_parts = []
-            raw_parts = []
             for line_pairs in example:
+                out_t_parts = []
+                raw_parts = []
                 for lraw, out_t_part in line_pairs:
-                    isp.push(lraw)
-                    out_t_parts.append(out_t_part)
-                    raw_parts.append(lraw)
+                    if out_t_part is not None:
+                        out_t_parts.append(out_t_part)
+                    
+                    if lraw is not None:
+                        isp.push(lraw)
+                        raw_parts.append(lraw)
 
                 out, out_raw = isp.source_raw_reset()
                 out_t = '\n'.join(out_t_parts).rstrip()
@@ -490,12 +481,10 b' class BlockIPythonInputTestCase(IPythonInputTestCase):'
             raw_parts = []
             out_t_parts = []
             for line_pairs in example:
-                for raw, out_t_part in line_pairs:
-                    raw_parts.append(raw)
-                    out_t_parts.append(out_t_part)
+                raw_parts, out_t_parts = zip(*line_pairs)
 
-                raw = '\n'.join(raw_parts)
-                out_t = '\n'.join(out_t_parts)
+                raw = '\n'.join(r for r in raw_parts if r is not None)
+                out_t = '\n'.join(o for o in out_t_parts if o is not None)
 
                 isp.push(raw)
                 out, out_raw = isp.source_raw_reset()
@@ -509,8 +498,8 b' class BlockIPythonInputTestCase(IPythonInputTestCase):'
 
             out_t_parts = []
             for line_pairs in example:
-                raw = '\n'.join(r for r, _ in line_pairs)
-                out_t = '\n'.join(t for _,t in line_pairs)
+                raw = '\n'.join(r for r, _ in line_pairs if r is not None)
+                out_t = '\n'.join(t for _,t in line_pairs if t is not None)
                 out = isp.transform_cell(raw)
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
@@ -595,9 +584,8 b' class CellMagicsCommon(object):'
         src = "%%cellm line\nbody\n"
         sp = self.sp
         sp.push(src)
-        nt.assert_equal(sp.cell_magic_parts, ['body\n'])
-        out = sp.source
-        ref = u"get_ipython()._run_cached_cell_magic({u}'cellm', {u}'line')\n"
+        out = sp.source_reset()
+        ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
         nt.assert_equal(out, py3compat.u_format(ref))
 
     def tearDown(self):

IPython/core/tests/test_inputtransformer.py

@@ -8,6 +8,7 b' u_fmt = py3compat.u_format'
 from IPython.core import inputtransformer as ipt
 
 def transform_and_reset(transformer):
+    transformer = transformer()
     def transform(inp):
         try:
             return transformer.push(inp)
@@ -19,6 +20,7 b' def transform_and_reset(transformer):'
 # Transformer tests
 def transform_checker(tests, transformer):
     """Utility to loop over test inputs"""
+    transformer = transformer()
     try:
         for inp, tr in tests:
             nt.assert_equal(transformer.push(inp), tr)
@@ -191,7 +193,8 b' syntax_ml = \\'
        
        leading_indent =
        [ [('    print "hi"','print "hi"'),
-          ('  for a in range(5):','for a in range(5):'),
+          ],
+         [('  for a in range(5):','for a in range(5):'),
           ('    a*2','  a*2'),
           ],
          [('    a="""','a="""'),
@@ -203,12 +206,12 b' syntax_ml = \\'
        ],
        
        cellmagic =
-       [ [('%%foo a', None),
-          (None, "get_ipython().run_cell_magic('foo', 'a', '')"),
+       [ [(u'%%foo a', None),
+          (None, u_fmt("get_ipython().run_cell_magic({u}'foo', {u}'a', {u}'')")),
           ],
-         [('%%bar 123', None),
-          ('hello', None),
-          ('', "get_ipython().run_cell_magic('bar', '123', 'hello')"),
+         [(u'%%bar 123', None),
+          (u'hello', None),
+          (u'', u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
           ],
        ],
        
@@ -223,22 +226,22 b' syntax_ml = \\'
        ],
        
        assign_magic =
-       [ [('a = %bc de \\', None),
-          ('fg', "a = get_ipython().magic('bc de  fg')"),
+       [ [(u'a = %bc de \\', None),
+          (u'fg', u_fmt("a = get_ipython().magic({u}'bc de  fg')")),
           ],
-         [('a = %bc de \\', None),
-          ('fg\\', None),
-          (None, "a = get_ipython().magic('bc de  fg')"),
+         [(u'a = %bc de \\', None),
+          (u'fg\\', None),
+          (None, u_fmt("a = get_ipython().magic({u}'bc de  fg')")),
           ],
        ],
        
        assign_system =
-       [ [('a = !bc de \\', None),
-          ('fg', "a = get_ipython().getoutput('bc de  fg')"),
+       [ [(u'a = !bc de \\', None),
+          (u'fg', u_fmt("a = get_ipython().getoutput({u}'bc de  fg')")),
           ],
-         [('a = !bc de \\', None),
-          ('fg\\', None),
-          (None, "a = get_ipython().getoutput('bc de  fg')"),
+         [(u'a = !bc de \\', None),
+          (u'fg\\', None),
+          (None, u_fmt("a = get_ipython().getoutput({u}'bc de  fg')")),
           ],
        ],
        )
@@ -259,6 +262,8 b' def test_classic_prompt():'
     tt.check_pairs(transform_and_reset(ipt.classic_prompt), syntax['classic_prompt'])
     for example in syntax_ml['classic_prompt']:
         transform_checker(example, ipt.classic_prompt)
+    for example in syntax_ml['multiline_datastructure']:
+        transform_checker(example, ipt.classic_prompt)
 
 
 def test_ipy_prompt():
@@ -303,3 +308,15 b' def test_escaped_multiline():'
 def test_cellmagic():
     for example in syntax_ml['cellmagic']:
         transform_checker(example, ipt.cellmagic)
+
+def test_has_comment():
+    tests = [('text', False),
+             ('text #comment', True),
+             ('text #comment\n', True),
+             ('#comment', True),
+             ('#comment\n', True),
+             ('a = "#string"', False),
+             ('a = "#string" # comment', True),
+             ('a #comment not "string"', True),
+             ]
+    tt.check_pairs(ipt.has_comment, tests)