test_tokenutil.py
141 lines
| 4.0 KiB
| text/x-python
|
PythonLexer
MinRK
|
r16578 | """Tests for tokenutil""" | ||
# Copyright (c) IPython Development Team. | ||||
# Distributed under the terms of the Modified BSD License. | ||||
import nose.tools as nt | ||||
Matthias Bussonnier
|
r26183 | import pytest | ||
from IPython.testing.decorators import skip_iptest_but_not_pytest | ||||
MinRK
|
r16578 | |||
Doug Blank
|
r18879 | from IPython.utils.tokenutil import token_at_cursor, line_at_cursor | ||
MinRK
|
r16578 | |||
MinRK
|
r16580 | def expect_token(expected, cell, cursor_pos): | ||
token = token_at_cursor(cell, cursor_pos) | ||||
offset = 0 | ||||
for line in cell.splitlines(): | ||||
if offset + len(line) >= cursor_pos: | ||||
break | ||||
else: | ||||
Matthias Bussonnier
|
r21655 | offset += len(line)+1 | ||
MinRK
|
r16580 | column = cursor_pos - offset | ||
line_with_cursor = '%s|%s' % (line[:column], line[column:]) | ||||
MinRK
|
r16578 | nt.assert_equal(token, expected, | ||
MinRK
|
r18453 | "Expected %r, got %r in: %r (pos %i)" % ( | ||
expected, token, line_with_cursor, cursor_pos) | ||||
MinRK
|
r16578 | ) | ||
def test_simple(): | ||||
cell = "foo" | ||||
for i in range(len(cell)): | ||||
expect_token("foo", cell, i) | ||||
def test_function(): | ||||
cell = "foo(a=5, b='10')" | ||||
expected = 'foo' | ||||
MinRK
|
r18453 | # up to `foo(|a=` | ||
for i in range(cell.find('a=') + 1): | ||||
expect_token("foo", cell, i) | ||||
# find foo after `=` | ||||
for i in [cell.find('=') + 1, cell.rfind('=') + 1]: | ||||
expect_token("foo", cell, i) | ||||
# in between `5,|` and `|b=` | ||||
for i in range(cell.find(','), cell.find('b=')): | ||||
MinRK
|
r16578 | expect_token("foo", cell, i) | ||
def test_multiline(): | ||||
cell = '\n'.join([ | ||||
'a = 5', | ||||
'b = hello("string", there)' | ||||
]) | ||||
expected = 'hello' | ||||
MinRK
|
r18453 | start = cell.index(expected) + 1 | ||
MinRK
|
r16580 | for i in range(start, start + len(expected)): | ||
expect_token(expected, cell, i) | ||||
Min RK
|
r20471 | expected = 'hello' | ||
MinRK
|
r18453 | start = cell.index(expected) + 1 | ||
MinRK
|
r16580 | for i in range(start, start + len(expected)): | ||
expect_token(expected, cell, i) | ||||
MinRK
|
r16578 | |||
Min RK
|
r21701 | def test_multiline_token(): | ||
cell = '\n'.join([ | ||||
'"""\n\nxxxxxxxxxx\n\n"""', | ||||
'5, """', | ||||
'docstring', | ||||
'multiline token', | ||||
'""", [', | ||||
'2, 3, "complicated"]', | ||||
'b = hello("string", there)' | ||||
]) | ||||
expected = 'hello' | ||||
start = cell.index(expected) + 1 | ||||
for i in range(start, start + len(expected)): | ||||
expect_token(expected, cell, i) | ||||
expected = 'hello' | ||||
start = cell.index(expected) + 1 | ||||
for i in range(start, start + len(expected)): | ||||
expect_token(expected, cell, i) | ||||
Min RK
|
r20471 | def test_nested_call(): | ||
cell = "foo(bar(a=5), b=10)" | ||||
expected = 'foo' | ||||
start = cell.index('bar') + 1 | ||||
for i in range(start, start + 3): | ||||
expect_token(expected, cell, i) | ||||
expected = 'bar' | ||||
start = cell.index('a=') | ||||
for i in range(start, start + 3): | ||||
expect_token(expected, cell, i) | ||||
expected = 'foo' | ||||
start = cell.index(')') + 1 | ||||
for i in range(start, len(cell)-1): | ||||
expect_token(expected, cell, i) | ||||
MinRK
|
r16578 | def test_attrs(): | ||
Min RK
|
r20471 | cell = "a = obj.attr.subattr" | ||
MinRK
|
r16578 | expected = 'obj' | ||
MinRK
|
r18453 | idx = cell.find('obj') + 1 | ||
MinRK
|
r16578 | for i in range(idx, idx + 3): | ||
expect_token(expected, cell, i) | ||||
MinRK
|
r18453 | idx = cell.find('.attr') + 2 | ||
MinRK
|
r16578 | expected = 'obj.attr' | ||
for i in range(idx, idx + 4): | ||||
expect_token(expected, cell, i) | ||||
MinRK
|
r18453 | idx = cell.find('.subattr') + 2 | ||
MinRK
|
r16578 | expected = 'obj.attr.subattr' | ||
for i in range(idx, len(cell)): | ||||
expect_token(expected, cell, i) | ||||
Doug Blank
|
r18879 | |||
def test_line_at_cursor(): | ||||
cell = "" | ||||
(line, offset) = line_at_cursor(cell, cursor_pos=11) | ||||
Thomas Kluyver
|
r23966 | nt.assert_equal(line, "") | ||
nt.assert_equal(offset, 0) | ||||
Matthias Bussonnier
|
r21655 | |||
Thomas Kluyver
|
r23959 | # The position after a newline should be the start of the following line. | ||
cell = "One\nTwo\n" | ||||
(line, offset) = line_at_cursor(cell, cursor_pos=4) | ||||
nt.assert_equal(line, "Two\n") | ||||
nt.assert_equal(offset, 4) | ||||
Thomas Kluyver
|
r23966 | # The end of a cell should be on the last line | ||
cell = "pri\npri" | ||||
(line, offset) = line_at_cursor(cell, cursor_pos=7) | ||||
nt.assert_equal(line, "pri") | ||||
nt.assert_equal(offset, 4) | ||||
Matthias Bussonnier
|
r26183 | |||
@pytest.mark.parametrize( | ||||
"c, token", | ||||
zip( | ||||
list(range(16, 22)) + list(range(22, 28)), | ||||
["int"] * (22 - 16) + ["map"] * (28 - 22), | ||||
), | ||||
) | ||||
@skip_iptest_but_not_pytest | ||||
def test_multiline_statement(c, token): | ||||
Matthias Bussonnier
|
r21655 | cell = """a = (1, | ||
3) | ||||
int() | ||||
map() | ||||
""" | ||||
Matthias Bussonnier
|
r26183 | expect_token(token, cell, c) | ||