From 93949c91f80c51856e069bb92532cc4d4910ef70 2010-07-22 20:00:42
From: epatters <epatters@enthought.com>
Date: 2010-07-22 20:00:42
Subject: [PATCH] * Fixed bug in CompletionLexer where a context would be generated for, e.g, 'foo('.
* Added unit test for this bug.
* Cleaned up and documented CompletionLexer.

---

diff --git a/IPython/frontend/qt/console/completion_lexer.py b/IPython/frontend/qt/console/completion_lexer.py
index 5eed553..55a9d19 100644
--- a/IPython/frontend/qt/console/completion_lexer.py
+++ b/IPython/frontend/qt/console/completion_lexer.py
@@ -13,6 +13,8 @@ class CompletionLexer(object):
                       'Python' : [ '.' ] }
 
     def __init__(self, lexer):
+        """ Create a CompletionLexer using the specified Pygments lexer.
+        """
         self.lexer = lexer
 
     def get_context(self, string):
@@ -23,21 +25,36 @@ class CompletionLexer(object):
         reversed_tokens = list(self._lexer.get_tokens(string))
         reversed_tokens.reverse()
 
-        # Pygments often tacks on a newline when none is specified in the input
+        # Pygments often tacks on a newline when none is specified in the input.
+        # Remove this newline.
         if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
                 not string.endswith('\n'):
             reversed_tokens.pop(0)
-
+        
         current_op = unicode()
         for token, text in reversed_tokens:
-            if is_token_subtype(token, Token.Name) and \
-                    (not context or current_op in self._name_separators):
-                if not context and current_op in self._name_separators:
-                    context.insert(0, unicode())
+
+            if is_token_subtype(token, Token.Name):
+
+                # Handle a trailing separator, e.g 'foo.bar.'
+                if current_op in self._name_separators:
+                    if not context:
+                        context.insert(0, unicode())
+
+                # Handle non-separator operators and punction.
+                elif current_op:
+                    break
+
                 context.insert(0, text)
                 current_op = unicode()
+
+            # Pygments doesn't understand that, e.g., '->' is a single operator
+            # in C++. This is why we have to build up an operator from
+            # potentially several tokens.
             elif token is Token.Operator or token is Token.Punctuation:
                 current_op = text + current_op
+
+            # Break on anything that is not a Operator, Punctuation, or Name.
             else:
                 break
 
diff --git a/IPython/frontend/qt/console/tests/test_completion_lexer.py b/IPython/frontend/qt/console/tests/test_completion_lexer.py
index 4ee37d5..ed958d8 100644
--- a/IPython/frontend/qt/console/tests/test_completion_lexer.py
+++ b/IPython/frontend/qt/console/tests/test_completion_lexer.py
@@ -15,14 +15,23 @@ class TestCompletionLexer(unittest.TestCase):
         """
         lexer = CompletionLexer(PythonLexer())
 
+        # Test simplest case.
         self.assertEquals(lexer.get_context("foo.bar.baz"),
                           [ "foo", "bar", "baz" ])
+
+        # Test trailing period.
         self.assertEquals(lexer.get_context("foo.bar."), [ "foo", "bar", "" ])
 
+        # Test with prompt present.
         self.assertEquals(lexer.get_context(">>> foo.bar.baz"),
                           [ "foo", "bar", "baz" ])
+
+        # Test spacing in name.
         self.assertEquals(lexer.get_context("foo.bar. baz"), [ "baz" ])
 
+        # Test parenthesis.
+        self.assertEquals(lexer.get_context("foo("), [])
+
     def testC(self):
         """ Does the CompletionLexer work for C/C++?
         """