From b8d87c257c0d8e9798712a5eb15380a0178804e7 2011-07-16 20:25:25
From: Thomas Kluyver <takowl@gmail.com>
Date: 2011-07-16 20:25:25
Subject: [PATCH] Handle tokenizing errors when checking if the ? at the end of the line is in a comment.

Closes gh-586

---

diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index 978f115..0ea51ae 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -173,7 +173,12 @@ def has_comment(src):
     Boolean: True if source has a comment.
     """
     readline = StringIO(src).readline
-    toktypes = set(t[0] for t in tokenize.generate_tokens(readline))
+    toktypes = set()
+    try:
+        for t in tokenize.generate_tokens(readline):
+            toktypes.add(t[0])
+    except tokenize.TokenError:
+        pass
     return(tokenize.COMMENT in toktypes)
 
 
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index 9b38ef8..77cbd2a 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -472,6 +472,7 @@ syntax = \
         ('a = abc?', 'get_ipython().magic(u"pinfo abc", next_input=u"a = abc")'),
         ('a = abc.qe??', 'get_ipython().magic(u"pinfo2 abc.qe", next_input=u"a = abc.qe")'),
         ('a = *.items?', 'get_ipython().magic(u"psearch *.items", next_input=u"a = *.items")'),
+        ('plot(a?', 'get_ipython().magic(u"pinfo a", next_input=u"plot(a")'),
         ('a*2 #comment?', 'a*2 #comment?'),
         ],