tokenutil.py
121 lines
| 3.4 KiB
| text/x-python
|
PythonLexer
|
r16578 | """Token-related utilities""" | ||
# Copyright (c) IPython Development Team. | ||||
# Distributed under the terms of the Modified BSD License. | ||||
from __future__ import absolute_import, print_function | ||||
from collections import namedtuple | ||||
from io import StringIO | ||||
from keyword import iskeyword | ||||
from . import tokenize2 | ||||
from .py3compat import cast_unicode_py2 | ||||
Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line']) | ||||
def generate_tokens(readline): | ||||
"""wrap generate_tokens to catch EOF errors""" | ||||
try: | ||||
for token in tokenize2.generate_tokens(readline): | ||||
yield token | ||||
except tokenize2.TokenError: | ||||
# catch EOF error | ||||
return | ||||
|
r18478 | def line_at_cursor(cell, cursor_pos=0): | ||
"""Return the line in a cell at a given cursor position | ||||
Used for calling line-based APIs that don't support multi-line input, yet. | ||||
Parameters | ||||
---------- | ||||
cell: text | ||||
multiline block of text | ||||
cursor_pos: integer | ||||
the cursor position | ||||
Returns | ||||
------- | ||||
(line, offset): (text, integer) | ||||
The line with the current cursor, and the character offset of the start of the line. | ||||
""" | ||||
offset = 0 | ||||
lines = cell.splitlines(True) | ||||
for line in lines: | ||||
next_offset = offset + len(line) | ||||
if next_offset >= cursor_pos: | ||||
break | ||||
offset = next_offset | ||||
|
r18879 | else: | ||
line = "" | ||||
|
r18478 | return (line, offset) | ||
|
r16580 | def token_at_cursor(cell, cursor_pos=0): | ||
|
r16578 | """Get the token at a given cursor | ||
Used for introspection. | ||||
|
r20471 | Function calls are prioritized, so the token for the callable will be returned | ||
if the cursor is anywhere inside the call. | ||||
|
r16578 | Parameters | ||
---------- | ||||
cell : unicode | ||||
A block of Python code | ||||
|
r16580 | cursor_pos : int | ||
The location of the cursor in the block where the token should be found | ||||
|
r16578 | """ | ||
cell = cast_unicode_py2(cell) | ||||
names = [] | ||||
tokens = [] | ||||
|
r16580 | offset = 0 | ||
|
r20471 | call_names = [] | ||
|
r16578 | for tup in generate_tokens(StringIO(cell).readline): | ||
tok = Token(*tup) | ||||
# token, text, start, end, line = tup | ||||
start_col = tok.start[1] | ||||
end_col = tok.end[1] | ||||
|
r18453 | # allow '|foo' to find 'foo' at the beginning of a line | ||
boundary = cursor_pos + 1 if start_col == 0 else cursor_pos | ||||
if offset + start_col >= boundary: | ||||
|
r16578 | # current token starts after the cursor, | ||
# don't consume it | ||||
break | ||||
if tok.token == tokenize2.NAME and not iskeyword(tok.text): | ||||
if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.': | ||||
names[-1] = "%s.%s" % (names[-1], tok.text) | ||||
else: | ||||
names.append(tok.text) | ||||
elif tok.token == tokenize2.OP: | ||||
if tok.text == '=' and names: | ||||
# don't inspect the lhs of an assignment | ||||
names.pop(-1) | ||||
|
r20471 | if tok.text == '(' and names: | ||
# if we are inside a function call, inspect the function | ||||
call_names.append(names[-1]) | ||||
elif tok.text == ')' and call_names: | ||||
call_names.pop(-1) | ||||
|
r16578 | |||
|
r16580 | if offset + end_col > cursor_pos: | ||
|
r16578 | # we found the cursor, stop reading | ||
break | ||||
tokens.append(tok) | ||||
|
r21822 | if tok.token in (tokenize2.NEWLINE, tokenize2.NL): | ||
|
r16580 | offset += len(tok.line) | ||
|
r16578 | |||
|
r20471 | if call_names: | ||
return call_names[-1] | ||||
elif names: | ||||
|
r16578 | return names[-1] | ||
else: | ||||
return '' | ||||