don't pick up tokens right of cursor

inspection of `foo(|a` was picking up 'a'
instead of 'foo' due to an off by one error

IPython/utils/tests/test_tokenutil.py

@@ -1,63 +1,70 b''
 """Tests for tokenutil"""
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import nose.tools as nt
 
 from IPython.utils.tokenutil import token_at_cursor
 
 def expect_token(expected, cell, cursor_pos):
     token = token_at_cursor(cell, cursor_pos)
     offset = 0
     for line in cell.splitlines():
         if offset + len(line) >= cursor_pos:
             break
         else:
             offset += len(line)
     column = cursor_pos - offset
     line_with_cursor = '%s|%s' % (line[:column], line[column:])
     line
     nt.assert_equal(token, expected,
-        "Excpected %r, got %r in: %s" % (
-        expected, token, line_with_cursor)
+        "Expected %r, got %r in: %r (pos %i)" % (
+        expected, token, line_with_cursor, cursor_pos)
     )
 
 def test_simple(): 
     cell = "foo"
     for i in range(len(cell)):
         expect_token("foo", cell, i)
 
 def test_function():
     cell = "foo(a=5, b='10')"
     expected = 'foo'
-    for i in (6,7,8,10,11,12):
+    # up to `foo(|a=`
+    for i in range(cell.find('a=') + 1):
+        expect_token("foo", cell, i)
+    # find foo after `=`
+    for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
+        expect_token("foo", cell, i)
+    # in between `5,|` and `|b=`
+    for i in range(cell.find(','), cell.find('b=')):
         expect_token("foo", cell, i)
 
 def test_multiline():
     cell = '\n'.join([
         'a = 5',
         'b = hello("string", there)'
     ])
     expected = 'hello'
-    start = cell.index(expected)
+    start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
     expected = 'there'
-    start = cell.index(expected)
+    start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
 
 def test_attrs():
     cell = "foo(a=obj.attr.subattr)"
     expected = 'obj'
-    idx = cell.find('obj')
+    idx = cell.find('obj') + 1
     for i in range(idx, idx + 3):
         expect_token(expected, cell, i)
-    idx = idx + 4
+    idx = cell.find('.attr') + 2
     expected = 'obj.attr'
     for i in range(idx, idx + 4):
         expect_token(expected, cell, i)
-    idx = idx + 5
+    idx = cell.find('.subattr') + 2
     expected = 'obj.attr.subattr'
     for i in range(idx, len(cell)):
         expect_token(expected, cell, i)

IPython/utils/tokenutil.py

@@ -1,78 +1,80 b''
 """Token-related utilities"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import absolute_import, print_function
 
 from collections import namedtuple
 from io import StringIO
 from keyword import iskeyword
 
 from . import tokenize2
 from .py3compat import cast_unicode_py2
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
 
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
         for token in tokenize2.generate_tokens(readline):
             yield token
     except tokenize2.TokenError:
         # catch EOF error
         return
 
 def token_at_cursor(cell, cursor_pos=0):
     """Get the token at a given cursor
     
     Used for introspection.
     
     Parameters
     ----------
     
     cell : unicode
         A block of Python code
     cursor_pos : int
         The location of the cursor in the block where the token should be found
     """
     cell = cast_unicode_py2(cell)
     names = []
     tokens = []
     offset = 0
     for tup in generate_tokens(StringIO(cell).readline):
         
         tok = Token(*tup)
         
         # token, text, start, end, line = tup
         start_col = tok.start[1]
         end_col = tok.end[1]
-        if offset + start_col > cursor_pos:
+        # allow '|foo' to find 'foo' at the beginning of a line
+        boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
+        if offset + start_col >= boundary:
             # current token starts after the cursor,
             # don't consume it
             break
         
         if tok.token == tokenize2.NAME and not iskeyword(tok.text):
             if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
         elif tok.token == tokenize2.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)
         
         if offset + end_col > cursor_pos:
             # we found the cursor, stop reading
             break
         
         tokens.append(tok)
         if tok.token == tokenize2.NEWLINE:
             offset += len(tok.line)
     
     if names:
         return names[-1]
     else:
         return ''