Show More
@@ -1,141 +1,143 b'' | |||||
1 | """Tests for tokenutil""" |
|
1 | """Tests for tokenutil""" | |
2 | # Copyright (c) IPython Development Team. |
|
2 | # Copyright (c) IPython Development Team. | |
3 | # Distributed under the terms of the Modified BSD License. |
|
3 | # Distributed under the terms of the Modified BSD License. | |
4 |
|
4 | |||
5 | import nose.tools as nt |
|
|||
6 | import pytest |
|
5 | import pytest | |
7 | from IPython.testing.decorators import skip_iptest_but_not_pytest |
|
6 | from IPython.testing.decorators import skip_iptest_but_not_pytest | |
8 |
|
7 | |||
9 | from IPython.utils.tokenutil import token_at_cursor, line_at_cursor |
|
8 | from IPython.utils.tokenutil import token_at_cursor, line_at_cursor | |
10 |
|
9 | |||
11 | def expect_token(expected, cell, cursor_pos): |
|
10 | def expect_token(expected, cell, cursor_pos): | |
12 | token = token_at_cursor(cell, cursor_pos) |
|
11 | token = token_at_cursor(cell, cursor_pos) | |
13 | offset = 0 |
|
12 | offset = 0 | |
14 | for line in cell.splitlines(): |
|
13 | for line in cell.splitlines(): | |
15 | if offset + len(line) >= cursor_pos: |
|
14 | if offset + len(line) >= cursor_pos: | |
16 | break |
|
15 | break | |
17 | else: |
|
16 | else: | |
18 | offset += len(line)+1 |
|
17 | offset += len(line)+1 | |
19 | column = cursor_pos - offset |
|
18 | column = cursor_pos - offset | |
20 |
line_with_cursor = |
|
19 | line_with_cursor = "%s|%s" % (line[:column], line[column:]) | |
21 | nt.assert_equal(token, expected, |
|
20 | assert token == expected, "Expected %r, got %r in: %r (pos %i)" % ( | |
22 | "Expected %r, got %r in: %r (pos %i)" % ( |
|
21 | expected, | |
23 | expected, token, line_with_cursor, cursor_pos) |
|
22 | token, | |
|
23 | line_with_cursor, | |||
|
24 | cursor_pos, | |||
24 | ) |
|
25 | ) | |
25 |
|
26 | |||
26 | def test_simple(): |
|
27 | ||
|
28 | def test_simple(): | |||
27 | cell = "foo" |
|
29 | cell = "foo" | |
28 | for i in range(len(cell)): |
|
30 | for i in range(len(cell)): | |
29 | expect_token("foo", cell, i) |
|
31 | expect_token("foo", cell, i) | |
30 |
|
32 | |||
31 | def test_function(): |
|
33 | def test_function(): | |
32 | cell = "foo(a=5, b='10')" |
|
34 | cell = "foo(a=5, b='10')" | |
33 | expected = 'foo' |
|
35 | expected = 'foo' | |
34 | # up to `foo(|a=` |
|
36 | # up to `foo(|a=` | |
35 | for i in range(cell.find('a=') + 1): |
|
37 | for i in range(cell.find('a=') + 1): | |
36 | expect_token("foo", cell, i) |
|
38 | expect_token("foo", cell, i) | |
37 | # find foo after `=` |
|
39 | # find foo after `=` | |
38 | for i in [cell.find('=') + 1, cell.rfind('=') + 1]: |
|
40 | for i in [cell.find('=') + 1, cell.rfind('=') + 1]: | |
39 | expect_token("foo", cell, i) |
|
41 | expect_token("foo", cell, i) | |
40 | # in between `5,|` and `|b=` |
|
42 | # in between `5,|` and `|b=` | |
41 | for i in range(cell.find(','), cell.find('b=')): |
|
43 | for i in range(cell.find(','), cell.find('b=')): | |
42 | expect_token("foo", cell, i) |
|
44 | expect_token("foo", cell, i) | |
43 |
|
45 | |||
44 | def test_multiline(): |
|
46 | def test_multiline(): | |
45 | cell = '\n'.join([ |
|
47 | cell = '\n'.join([ | |
46 | 'a = 5', |
|
48 | 'a = 5', | |
47 | 'b = hello("string", there)' |
|
49 | 'b = hello("string", there)' | |
48 | ]) |
|
50 | ]) | |
49 | expected = 'hello' |
|
51 | expected = 'hello' | |
50 | start = cell.index(expected) + 1 |
|
52 | start = cell.index(expected) + 1 | |
51 | for i in range(start, start + len(expected)): |
|
53 | for i in range(start, start + len(expected)): | |
52 | expect_token(expected, cell, i) |
|
54 | expect_token(expected, cell, i) | |
53 | expected = 'hello' |
|
55 | expected = 'hello' | |
54 | start = cell.index(expected) + 1 |
|
56 | start = cell.index(expected) + 1 | |
55 | for i in range(start, start + len(expected)): |
|
57 | for i in range(start, start + len(expected)): | |
56 | expect_token(expected, cell, i) |
|
58 | expect_token(expected, cell, i) | |
57 |
|
59 | |||
58 | def test_multiline_token(): |
|
60 | def test_multiline_token(): | |
59 | cell = '\n'.join([ |
|
61 | cell = '\n'.join([ | |
60 | '"""\n\nxxxxxxxxxx\n\n"""', |
|
62 | '"""\n\nxxxxxxxxxx\n\n"""', | |
61 | '5, """', |
|
63 | '5, """', | |
62 | 'docstring', |
|
64 | 'docstring', | |
63 | 'multiline token', |
|
65 | 'multiline token', | |
64 | '""", [', |
|
66 | '""", [', | |
65 | '2, 3, "complicated"]', |
|
67 | '2, 3, "complicated"]', | |
66 | 'b = hello("string", there)' |
|
68 | 'b = hello("string", there)' | |
67 | ]) |
|
69 | ]) | |
68 | expected = 'hello' |
|
70 | expected = 'hello' | |
69 | start = cell.index(expected) + 1 |
|
71 | start = cell.index(expected) + 1 | |
70 | for i in range(start, start + len(expected)): |
|
72 | for i in range(start, start + len(expected)): | |
71 | expect_token(expected, cell, i) |
|
73 | expect_token(expected, cell, i) | |
72 | expected = 'hello' |
|
74 | expected = 'hello' | |
73 | start = cell.index(expected) + 1 |
|
75 | start = cell.index(expected) + 1 | |
74 | for i in range(start, start + len(expected)): |
|
76 | for i in range(start, start + len(expected)): | |
75 | expect_token(expected, cell, i) |
|
77 | expect_token(expected, cell, i) | |
76 |
|
78 | |||
77 | def test_nested_call(): |
|
79 | def test_nested_call(): | |
78 | cell = "foo(bar(a=5), b=10)" |
|
80 | cell = "foo(bar(a=5), b=10)" | |
79 | expected = 'foo' |
|
81 | expected = 'foo' | |
80 | start = cell.index('bar') + 1 |
|
82 | start = cell.index('bar') + 1 | |
81 | for i in range(start, start + 3): |
|
83 | for i in range(start, start + 3): | |
82 | expect_token(expected, cell, i) |
|
84 | expect_token(expected, cell, i) | |
83 | expected = 'bar' |
|
85 | expected = 'bar' | |
84 | start = cell.index('a=') |
|
86 | start = cell.index('a=') | |
85 | for i in range(start, start + 3): |
|
87 | for i in range(start, start + 3): | |
86 | expect_token(expected, cell, i) |
|
88 | expect_token(expected, cell, i) | |
87 | expected = 'foo' |
|
89 | expected = 'foo' | |
88 | start = cell.index(')') + 1 |
|
90 | start = cell.index(')') + 1 | |
89 | for i in range(start, len(cell)-1): |
|
91 | for i in range(start, len(cell)-1): | |
90 | expect_token(expected, cell, i) |
|
92 | expect_token(expected, cell, i) | |
91 |
|
93 | |||
92 | def test_attrs(): |
|
94 | def test_attrs(): | |
93 | cell = "a = obj.attr.subattr" |
|
95 | cell = "a = obj.attr.subattr" | |
94 | expected = 'obj' |
|
96 | expected = 'obj' | |
95 | idx = cell.find('obj') + 1 |
|
97 | idx = cell.find('obj') + 1 | |
96 | for i in range(idx, idx + 3): |
|
98 | for i in range(idx, idx + 3): | |
97 | expect_token(expected, cell, i) |
|
99 | expect_token(expected, cell, i) | |
98 | idx = cell.find('.attr') + 2 |
|
100 | idx = cell.find('.attr') + 2 | |
99 | expected = 'obj.attr' |
|
101 | expected = 'obj.attr' | |
100 | for i in range(idx, idx + 4): |
|
102 | for i in range(idx, idx + 4): | |
101 | expect_token(expected, cell, i) |
|
103 | expect_token(expected, cell, i) | |
102 | idx = cell.find('.subattr') + 2 |
|
104 | idx = cell.find('.subattr') + 2 | |
103 | expected = 'obj.attr.subattr' |
|
105 | expected = 'obj.attr.subattr' | |
104 | for i in range(idx, len(cell)): |
|
106 | for i in range(idx, len(cell)): | |
105 | expect_token(expected, cell, i) |
|
107 | expect_token(expected, cell, i) | |
106 |
|
108 | |||
107 | def test_line_at_cursor(): |
|
109 | def test_line_at_cursor(): | |
108 | cell = "" |
|
110 | cell = "" | |
109 | (line, offset) = line_at_cursor(cell, cursor_pos=11) |
|
111 | (line, offset) = line_at_cursor(cell, cursor_pos=11) | |
110 |
|
|
112 | assert line == "" | |
111 |
|
|
113 | assert offset == 0 | |
112 |
|
114 | |||
113 | # The position after a newline should be the start of the following line. |
|
115 | # The position after a newline should be the start of the following line. | |
114 | cell = "One\nTwo\n" |
|
116 | cell = "One\nTwo\n" | |
115 | (line, offset) = line_at_cursor(cell, cursor_pos=4) |
|
117 | (line, offset) = line_at_cursor(cell, cursor_pos=4) | |
116 |
|
|
118 | assert line == "Two\n" | |
117 |
|
|
119 | assert offset == 4 | |
118 |
|
120 | |||
119 | # The end of a cell should be on the last line |
|
121 | # The end of a cell should be on the last line | |
120 | cell = "pri\npri" |
|
122 | cell = "pri\npri" | |
121 | (line, offset) = line_at_cursor(cell, cursor_pos=7) |
|
123 | (line, offset) = line_at_cursor(cell, cursor_pos=7) | |
122 |
|
|
124 | assert line == "pri" | |
123 |
|
|
125 | assert offset == 4 | |
124 |
|
126 | |||
125 |
|
127 | |||
126 | @pytest.mark.parametrize( |
|
128 | @pytest.mark.parametrize( | |
127 | "c, token", |
|
129 | "c, token", | |
128 | zip( |
|
130 | zip( | |
129 | list(range(16, 22)) + list(range(22, 28)), |
|
131 | list(range(16, 22)) + list(range(22, 28)), | |
130 | ["int"] * (22 - 16) + ["map"] * (28 - 22), |
|
132 | ["int"] * (22 - 16) + ["map"] * (28 - 22), | |
131 | ), |
|
133 | ), | |
132 | ) |
|
134 | ) | |
133 | @skip_iptest_but_not_pytest |
|
135 | @skip_iptest_but_not_pytest | |
134 | def test_multiline_statement(c, token): |
|
136 | def test_multiline_statement(c, token): | |
135 | cell = """a = (1, |
|
137 | cell = """a = (1, | |
136 | 3) |
|
138 | 3) | |
137 |
|
139 | |||
138 | int() |
|
140 | int() | |
139 | map() |
|
141 | map() | |
140 | """ |
|
142 | """ | |
141 | expect_token(token, cell, c) |
|
143 | expect_token(token, cell, c) |
General Comments 0
You need to be logged in to leave comments.
Login now