* Fixed bug in CompletionLexer where a context would be generated for, e.g, 'foo('.

* Added unit test for this bug.
* Cleaned up and documented CompletionLexer.

IPython/frontend/qt/console/completion_lexer.py

@@ -1,57 +1,74 b''
 # System library imports
 from pygments.token import Token, is_token_subtype
 
 
 class CompletionLexer(object):
     """ Uses Pygments and some auxillary information to lex code snippets for 
         symbol contexts.
     """
 
     # Maps Lexer names to a list of possible name separators
     separator_map = { 'C' : [ '.', '->' ],
                       'C++' : [ '.', '->', '::' ],
                       'Python' : [ '.' ] }
 
     def __init__(self, lexer):
+        """ Create a CompletionLexer using the specified Pygments lexer.
+        """
         self.lexer = lexer
 
     def get_context(self, string):
         """ Assuming the cursor is at the end of the specified string, get the
             context (a list of names) for the symbol at cursor position.
         """
         context = []
         reversed_tokens = list(self._lexer.get_tokens(string))
         reversed_tokens.reverse()
 
-        # Pygments often tacks on a newline when none is specified in the input
+        # Pygments often tacks on a newline when none is specified in the input.
+        # Remove this newline.
         if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
                 not string.endswith('\n'):
             reversed_tokens.pop(0)
-
+        
         current_op = unicode()
         for token, text in reversed_tokens:
-            if is_token_subtype(token, Token.Name) and \
-                    (not context or current_op in self._name_separators):
-                if not context and current_op in self._name_separators:
-                    context.insert(0, unicode())
+
+            if is_token_subtype(token, Token.Name):
+
+                # Handle a trailing separator, e.g 'foo.bar.'
+                if current_op in self._name_separators:
+                    if not context:
+                        context.insert(0, unicode())
+
+                # Handle non-separator operators and punction.
+                elif current_op:
+                    break
+
                 context.insert(0, text)
                 current_op = unicode()
+
+            # Pygments doesn't understand that, e.g., '->' is a single operator
+            # in C++. This is why we have to build up an operator from
+            # potentially several tokens.
             elif token is Token.Operator or token is Token.Punctuation:
                 current_op = text + current_op
+
+            # Break on anything that is not a Operator, Punctuation, or Name.
             else:
                 break
 
         return context
 
     def get_lexer(self, lexer):
         return self._lexer
 
     def set_lexer(self, lexer, name_separators=None):
         self._lexer = lexer
         if name_separators is None:
             self._name_separators = self.separator_map.get(lexer.name, ['.'])
         else:
             self._name_separators = list(name_separators)
 
     lexer = property(get_lexer, set_lexer)
     

IPython/frontend/qt/console/tests/test_completion_lexer.py

@@ -1,38 +1,47 b''
 # Standard library imports
 import unittest
 
 # System library imports
 from pygments.lexers import CLexer, CppLexer, PythonLexer
 
 # Local imports
 from IPython.frontend.qt.console.completion_lexer import CompletionLexer
 
 
 class TestCompletionLexer(unittest.TestCase):
     
     def testPython(self):
         """ Does the CompletionLexer work for Python?
         """
         lexer = CompletionLexer(PythonLexer())
 
+        # Test simplest case.
         self.assertEquals(lexer.get_context("foo.bar.baz"),
                           [ "foo", "bar", "baz" ])
+
+        # Test trailing period.
         self.assertEquals(lexer.get_context("foo.bar."), [ "foo", "bar", "" ])
 
+        # Test with prompt present.
         self.assertEquals(lexer.get_context(">>> foo.bar.baz"),
                           [ "foo", "bar", "baz" ])
+
+        # Test spacing in name.
         self.assertEquals(lexer.get_context("foo.bar. baz"), [ "baz" ])
 
+        # Test parenthesis.
+        self.assertEquals(lexer.get_context("foo("), [])
+
     def testC(self):
         """ Does the CompletionLexer work for C/C++?
         """
         lexer = CompletionLexer(CLexer())
         self.assertEquals(lexer.get_context("foo.bar"), [ "foo", "bar" ])
         self.assertEquals(lexer.get_context("foo->bar"), [ "foo", "bar" ])
 
         lexer = CompletionLexer(CppLexer())
         self.assertEquals(lexer.get_context("Foo::Bar"), [ "Foo", "Bar" ])
 
 
 if __name__ == '__main__':
     unittest.main()