TAB on empty line causes crash; with test

IPython/utils/tests/test_tokenutil.py

@@ -1,70 +1,76 b''
 """Tests for tokenutil"""
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import nose.tools as nt
 
-from IPython.utils.tokenutil import token_at_cursor
+from IPython.utils.tokenutil import token_at_cursor, line_at_cursor
 
 def expect_token(expected, cell, cursor_pos):
     token = token_at_cursor(cell, cursor_pos)
     offset = 0
     for line in cell.splitlines():
         if offset + len(line) >= cursor_pos:
             break
         else:
             offset += len(line)
     column = cursor_pos - offset
     line_with_cursor = '%s|%s' % (line[:column], line[column:])
     line
     nt.assert_equal(token, expected,
         "Expected %r, got %r in: %r (pos %i)" % (
         expected, token, line_with_cursor, cursor_pos)
     )
 
 def test_simple(): 
     cell = "foo"
     for i in range(len(cell)):
         expect_token("foo", cell, i)
 
 def test_function():
     cell = "foo(a=5, b='10')"
     expected = 'foo'
     # up to `foo(|a=`
     for i in range(cell.find('a=') + 1):
         expect_token("foo", cell, i)
     # find foo after `=`
     for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
         expect_token("foo", cell, i)
     # in between `5,|` and `|b=`
     for i in range(cell.find(','), cell.find('b=')):
         expect_token("foo", cell, i)
 
 def test_multiline():
     cell = '\n'.join([
         'a = 5',
         'b = hello("string", there)'
     ])
     expected = 'hello'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
     expected = 'there'
     start = cell.index(expected) + 1
     for i in range(start, start + len(expected)):
         expect_token(expected, cell, i)
 
 def test_attrs():
     cell = "foo(a=obj.attr.subattr)"
     expected = 'obj'
     idx = cell.find('obj') + 1
     for i in range(idx, idx + 3):
         expect_token(expected, cell, i)
     idx = cell.find('.attr') + 2
     expected = 'obj.attr'
     for i in range(idx, idx + 4):
         expect_token(expected, cell, i)
     idx = cell.find('.subattr') + 2
     expected = 'obj.attr.subattr'
     for i in range(idx, len(cell)):
         expect_token(expected, cell, i)
+
+def test_line_at_cursor():
+    cell = ""
+    (line, offset) = line_at_cursor(cell, cursor_pos=11)
+    assert line == "", ("Expected '', got %r" % line)
+    assert offset == 0, ("Expected '', got %r" % line)

IPython/utils/tokenutil.py

@@ -1,108 +1,110 b''
 """Token-related utilities"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import absolute_import, print_function
 
 from collections import namedtuple
 from io import StringIO
 from keyword import iskeyword
 
 from . import tokenize2
 from .py3compat import cast_unicode_py2
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
 
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
         for token in tokenize2.generate_tokens(readline):
             yield token
     except tokenize2.TokenError:
         # catch EOF error
         return
 
 def line_at_cursor(cell, cursor_pos=0):
     """Return the line in a cell at a given cursor position
     
     Used for calling line-based APIs that don't support multi-line input, yet.
     
     Parameters
     ----------
     
     cell: text
         multiline block of text
     cursor_pos: integer
         the cursor position
     
     Returns
     -------
     
     (line, offset): (text, integer)
         The line with the current cursor, and the character offset of the start of the line.
     """
     offset = 0
     lines = cell.splitlines(True)
     for line in lines:
         next_offset = offset + len(line)
         if next_offset >= cursor_pos:
             break
         offset = next_offset
+    else:
+        line = ""
     return (line, offset)
 
 def token_at_cursor(cell, cursor_pos=0):
     """Get the token at a given cursor
     
     Used for introspection.
     
     Parameters
     ----------
     
     cell : unicode
         A block of Python code
     cursor_pos : int
         The location of the cursor in the block where the token should be found
     """
     cell = cast_unicode_py2(cell)
     names = []
     tokens = []
     offset = 0
     for tup in generate_tokens(StringIO(cell).readline):
         
         tok = Token(*tup)
         
         # token, text, start, end, line = tup
         start_col = tok.start[1]
         end_col = tok.end[1]
         # allow '|foo' to find 'foo' at the beginning of a line
         boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
         if offset + start_col >= boundary:
             # current token starts after the cursor,
             # don't consume it
             break
         
         if tok.token == tokenize2.NAME and not iskeyword(tok.text):
             if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
         elif tok.token == tokenize2.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)
         
         if offset + end_col > cursor_pos:
             # we found the cursor, stop reading
             break
         
         tokens.append(tok)
         if tok.token == tokenize2.NEWLINE:
             offset += len(tok.line)
     
     if names:
         return names[-1]
     else:
         return ''