IPython/utils/tokenutil.py

"""Token-related utilities"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import absolute_import, print_function

from collections import namedtuple
from io import StringIO
from keyword import iskeyword

from . import tokenize2
from .py3compat import cast_unicode_py2

Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])

def generate_tokens(readline):
    """wrap generate_tokens to catch EOF errors"""
    try:
        for token in tokenize2.generate_tokens(readline):
            yield token
    except tokenize2.TokenError:
        # catch EOF error
        return

def token_at_cursor(cell, cursor_pos=0):
    """Get the token at a given cursor
    
    Used for introspection.
    
    Parameters
    ----------
    
    cell : unicode
        A block of Python code
    cursor_pos : int
        The location of the cursor in the block where the token should be found
    """
    cell = cast_unicode_py2(cell)
    names = []
    tokens = []
    offset = 0
    for tup in generate_tokens(StringIO(cell).readline):
        
        tok = Token(*tup)
        
        # token, text, start, end, line = tup
        start_col = tok.start[1]
        end_col = tok.end[1]
        if offset + start_col > cursor_pos:
            # current token starts after the cursor,
            # don't consume it
            break
        
        if tok.token == tokenize2.NAME and not iskeyword(tok.text):
            if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
                names[-1] = "%s.%s" % (names[-1], tok.text)
            else:
                names.append(tok.text)
        elif tok.token == tokenize2.OP:
            if tok.text == '=' and names:
                # don't inspect the lhs of an assignment
                names.pop(-1)
        
        if offset + end_col > cursor_pos:
            # we found the cursor, stop reading
            break
        
        tokens.append(tok)
        if tok.token == tokenize2.NEWLINE:
            offset += len(tok.line)
    
    if names:
        return names[-1]
    else:
        return ''