diff --git a/IPython/utils/tests/test_tokenutil.py b/IPython/utils/tests/test_tokenutil.py new file mode 100644 index 0000000..69e3362 --- /dev/null +++ b/IPython/utils/tests/test_tokenutil.py @@ -0,0 +1,56 @@ +"""Tests for tokenutil""" +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import nose.tools as nt + +from IPython.utils.tokenutil import token_at_cursor + +def expect_token(expected, cell, column, line=0): + token = token_at_cursor(cell, column, line) + + lines = cell.splitlines() + line_with_cursor = '%s|%s' % (lines[line][:column], lines[line][column:]) + line + nt.assert_equal(token, expected, + "Excpected %r, got %r in: %s" % ( + expected, token, line_with_cursor) + ) + +def test_simple(): + cell = "foo" + for i in range(len(cell)): + expect_token("foo", cell, i) + +def test_function(): + cell = "foo(a=5, b='10')" + expected = 'foo' + for i in (6,7,8,10,11,12): + expect_token("foo", cell, i) + +def test_multiline(): + cell = '\n'.join([ + 'a = 5', + 'b = hello("string", there)' + ]) + expected = 'hello' + for i in range(4,9): + expect_token(expected, cell, i, 1) + expected = 'there' + for i in range(21,27): + expect_token(expected, cell, i, 1) + +def test_attrs(): + cell = "foo(a=obj.attr.subattr)" + expected = 'obj' + idx = cell.find('obj') + for i in range(idx, idx + 3): + expect_token(expected, cell, i) + idx = idx + 4 + expected = 'obj.attr' + for i in range(idx, idx + 4): + expect_token(expected, cell, i) + idx = idx + 5 + expected = 'obj.attr.subattr' + for i in range(idx, len(cell)): + expect_token(expected, cell, i) diff --git a/IPython/utils/tokenutil.py b/IPython/utils/tokenutil.py new file mode 100644 index 0000000..6c1fbac --- /dev/null +++ b/IPython/utils/tokenutil.py @@ -0,0 +1,80 @@ +"""Token-related utilities""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +from __future__ import absolute_import, print_function + +from collections import namedtuple +from io import StringIO +from keyword import iskeyword + +from . import tokenize2 +from .py3compat import cast_unicode_py2 + +Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line']) + +def generate_tokens(readline): + """wrap generate_tokens to catch EOF errors""" + try: + for token in tokenize2.generate_tokens(readline): + yield token + except tokenize2.TokenError: + # catch EOF error + return + +def token_at_cursor(cell, column, line=0): + """Get the token at a given cursor + + Used for introspection. + + Parameters + ---------- + + cell : unicode + A block of Python code + column : int + The column of the cursor offset, where the token should be found + line : int, optional + The line where the token should be found (optional if cell is a single line) + """ + cell = cast_unicode_py2(cell) + names = [] + tokens = [] + current_line = 0 + for tup in generate_tokens(StringIO(cell).readline): + + tok = Token(*tup) + + # token, text, start, end, line = tup + start_col = tok.start[1] + end_col = tok.end[1] + if line == current_line and start_col > column: + # current token starts after the cursor, + # don't consume it + break + + if tok.token == tokenize2.NAME and not iskeyword(tok.text): + if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.': + names[-1] = "%s.%s" % (names[-1], tok.text) + else: + names.append(tok.text) + elif tok.token == tokenize2.OP: + if tok.text == '=' and names: + # don't inspect the lhs of an assignment + names.pop(-1) + + if line == current_line and end_col > column: + # we found the cursor, stop reading + break + + tokens.append(tok) + if tok.token == tokenize2.NEWLINE: + current_line += 1 + + if names: + return names[-1] + else: + return '' + +