##// END OF EJS Templates
Identify position after a newline as the start of the next line...
Thomas Kluyver -
Show More
@@ -1,121 +1,127 b''
1 """Tests for tokenutil"""
1 """Tests for tokenutil"""
2 # Copyright (c) IPython Development Team.
2 # Copyright (c) IPython Development Team.
3 # Distributed under the terms of the Modified BSD License.
3 # Distributed under the terms of the Modified BSD License.
4
4
5 import nose.tools as nt
5 import nose.tools as nt
6
6
7 from IPython.utils.tokenutil import token_at_cursor, line_at_cursor
7 from IPython.utils.tokenutil import token_at_cursor, line_at_cursor
8
8
9 def expect_token(expected, cell, cursor_pos):
9 def expect_token(expected, cell, cursor_pos):
10 token = token_at_cursor(cell, cursor_pos)
10 token = token_at_cursor(cell, cursor_pos)
11 offset = 0
11 offset = 0
12 for line in cell.splitlines():
12 for line in cell.splitlines():
13 if offset + len(line) >= cursor_pos:
13 if offset + len(line) >= cursor_pos:
14 break
14 break
15 else:
15 else:
16 offset += len(line)+1
16 offset += len(line)+1
17 column = cursor_pos - offset
17 column = cursor_pos - offset
18 line_with_cursor = '%s|%s' % (line[:column], line[column:])
18 line_with_cursor = '%s|%s' % (line[:column], line[column:])
19 nt.assert_equal(token, expected,
19 nt.assert_equal(token, expected,
20 "Expected %r, got %r in: %r (pos %i)" % (
20 "Expected %r, got %r in: %r (pos %i)" % (
21 expected, token, line_with_cursor, cursor_pos)
21 expected, token, line_with_cursor, cursor_pos)
22 )
22 )
23
23
24 def test_simple():
24 def test_simple():
25 cell = "foo"
25 cell = "foo"
26 for i in range(len(cell)):
26 for i in range(len(cell)):
27 expect_token("foo", cell, i)
27 expect_token("foo", cell, i)
28
28
29 def test_function():
29 def test_function():
30 cell = "foo(a=5, b='10')"
30 cell = "foo(a=5, b='10')"
31 expected = 'foo'
31 expected = 'foo'
32 # up to `foo(|a=`
32 # up to `foo(|a=`
33 for i in range(cell.find('a=') + 1):
33 for i in range(cell.find('a=') + 1):
34 expect_token("foo", cell, i)
34 expect_token("foo", cell, i)
35 # find foo after `=`
35 # find foo after `=`
36 for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
36 for i in [cell.find('=') + 1, cell.rfind('=') + 1]:
37 expect_token("foo", cell, i)
37 expect_token("foo", cell, i)
38 # in between `5,|` and `|b=`
38 # in between `5,|` and `|b=`
39 for i in range(cell.find(','), cell.find('b=')):
39 for i in range(cell.find(','), cell.find('b=')):
40 expect_token("foo", cell, i)
40 expect_token("foo", cell, i)
41
41
42 def test_multiline():
42 def test_multiline():
43 cell = '\n'.join([
43 cell = '\n'.join([
44 'a = 5',
44 'a = 5',
45 'b = hello("string", there)'
45 'b = hello("string", there)'
46 ])
46 ])
47 expected = 'hello'
47 expected = 'hello'
48 start = cell.index(expected) + 1
48 start = cell.index(expected) + 1
49 for i in range(start, start + len(expected)):
49 for i in range(start, start + len(expected)):
50 expect_token(expected, cell, i)
50 expect_token(expected, cell, i)
51 expected = 'hello'
51 expected = 'hello'
52 start = cell.index(expected) + 1
52 start = cell.index(expected) + 1
53 for i in range(start, start + len(expected)):
53 for i in range(start, start + len(expected)):
54 expect_token(expected, cell, i)
54 expect_token(expected, cell, i)
55
55
56 def test_multiline_token():
56 def test_multiline_token():
57 cell = '\n'.join([
57 cell = '\n'.join([
58 '"""\n\nxxxxxxxxxx\n\n"""',
58 '"""\n\nxxxxxxxxxx\n\n"""',
59 '5, """',
59 '5, """',
60 'docstring',
60 'docstring',
61 'multiline token',
61 'multiline token',
62 '""", [',
62 '""", [',
63 '2, 3, "complicated"]',
63 '2, 3, "complicated"]',
64 'b = hello("string", there)'
64 'b = hello("string", there)'
65 ])
65 ])
66 expected = 'hello'
66 expected = 'hello'
67 start = cell.index(expected) + 1
67 start = cell.index(expected) + 1
68 for i in range(start, start + len(expected)):
68 for i in range(start, start + len(expected)):
69 expect_token(expected, cell, i)
69 expect_token(expected, cell, i)
70 expected = 'hello'
70 expected = 'hello'
71 start = cell.index(expected) + 1
71 start = cell.index(expected) + 1
72 for i in range(start, start + len(expected)):
72 for i in range(start, start + len(expected)):
73 expect_token(expected, cell, i)
73 expect_token(expected, cell, i)
74
74
75 def test_nested_call():
75 def test_nested_call():
76 cell = "foo(bar(a=5), b=10)"
76 cell = "foo(bar(a=5), b=10)"
77 expected = 'foo'
77 expected = 'foo'
78 start = cell.index('bar') + 1
78 start = cell.index('bar') + 1
79 for i in range(start, start + 3):
79 for i in range(start, start + 3):
80 expect_token(expected, cell, i)
80 expect_token(expected, cell, i)
81 expected = 'bar'
81 expected = 'bar'
82 start = cell.index('a=')
82 start = cell.index('a=')
83 for i in range(start, start + 3):
83 for i in range(start, start + 3):
84 expect_token(expected, cell, i)
84 expect_token(expected, cell, i)
85 expected = 'foo'
85 expected = 'foo'
86 start = cell.index(')') + 1
86 start = cell.index(')') + 1
87 for i in range(start, len(cell)-1):
87 for i in range(start, len(cell)-1):
88 expect_token(expected, cell, i)
88 expect_token(expected, cell, i)
89
89
90 def test_attrs():
90 def test_attrs():
91 cell = "a = obj.attr.subattr"
91 cell = "a = obj.attr.subattr"
92 expected = 'obj'
92 expected = 'obj'
93 idx = cell.find('obj') + 1
93 idx = cell.find('obj') + 1
94 for i in range(idx, idx + 3):
94 for i in range(idx, idx + 3):
95 expect_token(expected, cell, i)
95 expect_token(expected, cell, i)
96 idx = cell.find('.attr') + 2
96 idx = cell.find('.attr') + 2
97 expected = 'obj.attr'
97 expected = 'obj.attr'
98 for i in range(idx, idx + 4):
98 for i in range(idx, idx + 4):
99 expect_token(expected, cell, i)
99 expect_token(expected, cell, i)
100 idx = cell.find('.subattr') + 2
100 idx = cell.find('.subattr') + 2
101 expected = 'obj.attr.subattr'
101 expected = 'obj.attr.subattr'
102 for i in range(idx, len(cell)):
102 for i in range(idx, len(cell)):
103 expect_token(expected, cell, i)
103 expect_token(expected, cell, i)
104
104
105 def test_line_at_cursor():
105 def test_line_at_cursor():
106 cell = ""
106 cell = ""
107 (line, offset) = line_at_cursor(cell, cursor_pos=11)
107 (line, offset) = line_at_cursor(cell, cursor_pos=11)
108 assert line == "", ("Expected '', got %r" % line)
108 assert line == "", ("Expected '', got %r" % line)
109 assert offset == 0, ("Expected '', got %r" % line)
109 assert offset == 0, ("Expected '', got %r" % line)
110
110
111 # The position after a newline should be the start of the following line.
112 cell = "One\nTwo\n"
113 (line, offset) = line_at_cursor(cell, cursor_pos=4)
114 nt.assert_equal(line, "Two\n")
115 nt.assert_equal(offset, 4)
116
111 def test_muliline_statement():
117 def test_muliline_statement():
112 cell = """a = (1,
118 cell = """a = (1,
113 3)
119 3)
114
120
115 int()
121 int()
116 map()
122 map()
117 """
123 """
118 for c in range(16, 22):
124 for c in range(16, 22):
119 yield lambda: expect_token("int", cell, c)
125 yield lambda: expect_token("int", cell, c)
120 for c in range(22, 28):
126 for c in range(22, 28):
121 yield lambda: expect_token("map", cell, c)
127 yield lambda: expect_token("map", cell, c)
@@ -1,125 +1,125 b''
1 """Token-related utilities"""
1 """Token-related utilities"""
2
2
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 from collections import namedtuple
6 from collections import namedtuple
7 from io import StringIO
7 from io import StringIO
8 from keyword import iskeyword
8 from keyword import iskeyword
9
9
10 from . import tokenize2
10 from . import tokenize2
11
11
12
12
13 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
13 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
14
14
15 def generate_tokens(readline):
15 def generate_tokens(readline):
16 """wrap generate_tokens to catch EOF errors"""
16 """wrap generate_tokens to catch EOF errors"""
17 try:
17 try:
18 for token in tokenize2.generate_tokens(readline):
18 for token in tokenize2.generate_tokens(readline):
19 yield token
19 yield token
20 except tokenize2.TokenError:
20 except tokenize2.TokenError:
21 # catch EOF error
21 # catch EOF error
22 return
22 return
23
23
24 def line_at_cursor(cell, cursor_pos=0):
24 def line_at_cursor(cell, cursor_pos=0):
25 """Return the line in a cell at a given cursor position
25 """Return the line in a cell at a given cursor position
26
26
27 Used for calling line-based APIs that don't support multi-line input, yet.
27 Used for calling line-based APIs that don't support multi-line input, yet.
28
28
29 Parameters
29 Parameters
30 ----------
30 ----------
31
31
32 cell: str
32 cell: str
33 multiline block of text
33 multiline block of text
34 cursor_pos: integer
34 cursor_pos: integer
35 the cursor position
35 the cursor position
36
36
37 Returns
37 Returns
38 -------
38 -------
39
39
40 (line, offset): (string, integer)
40 (line, offset): (string, integer)
41 The line with the current cursor, and the character offset of the start of the line.
41 The line with the current cursor, and the character offset of the start of the line.
42 """
42 """
43 offset = 0
43 offset = 0
44 lines = cell.splitlines(True)
44 lines = cell.splitlines(True)
45 for line in lines:
45 for line in lines:
46 next_offset = offset + len(line)
46 next_offset = offset + len(line)
47 if next_offset >= cursor_pos:
47 if next_offset > cursor_pos:
48 break
48 break
49 offset = next_offset
49 offset = next_offset
50 else:
50 else:
51 line = ""
51 line = ""
52 return (line, offset)
52 return (line, offset)
53
53
54 def token_at_cursor(cell, cursor_pos=0):
54 def token_at_cursor(cell, cursor_pos=0):
55 """Get the token at a given cursor
55 """Get the token at a given cursor
56
56
57 Used for introspection.
57 Used for introspection.
58
58
59 Function calls are prioritized, so the token for the callable will be returned
59 Function calls are prioritized, so the token for the callable will be returned
60 if the cursor is anywhere inside the call.
60 if the cursor is anywhere inside the call.
61
61
62 Parameters
62 Parameters
63 ----------
63 ----------
64
64
65 cell : unicode
65 cell : unicode
66 A block of Python code
66 A block of Python code
67 cursor_pos : int
67 cursor_pos : int
68 The location of the cursor in the block where the token should be found
68 The location of the cursor in the block where the token should be found
69 """
69 """
70 names = []
70 names = []
71 tokens = []
71 tokens = []
72 call_names = []
72 call_names = []
73
73
74 offsets = {1: 0} # lines start at 1
74 offsets = {1: 0} # lines start at 1
75 for tup in generate_tokens(StringIO(cell).readline):
75 for tup in generate_tokens(StringIO(cell).readline):
76
76
77 tok = Token(*tup)
77 tok = Token(*tup)
78
78
79 # token, text, start, end, line = tup
79 # token, text, start, end, line = tup
80 start_line, start_col = tok.start
80 start_line, start_col = tok.start
81 end_line, end_col = tok.end
81 end_line, end_col = tok.end
82 if end_line + 1 not in offsets:
82 if end_line + 1 not in offsets:
83 # keep track of offsets for each line
83 # keep track of offsets for each line
84 lines = tok.line.splitlines(True)
84 lines = tok.line.splitlines(True)
85 for lineno, line in enumerate(lines, start_line + 1):
85 for lineno, line in enumerate(lines, start_line + 1):
86 if lineno not in offsets:
86 if lineno not in offsets:
87 offsets[lineno] = offsets[lineno-1] + len(line)
87 offsets[lineno] = offsets[lineno-1] + len(line)
88
88
89 offset = offsets[start_line]
89 offset = offsets[start_line]
90 # allow '|foo' to find 'foo' at the beginning of a line
90 # allow '|foo' to find 'foo' at the beginning of a line
91 boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
91 boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
92 if offset + start_col >= boundary:
92 if offset + start_col >= boundary:
93 # current token starts after the cursor,
93 # current token starts after the cursor,
94 # don't consume it
94 # don't consume it
95 break
95 break
96
96
97 if tok.token == tokenize2.NAME and not iskeyword(tok.text):
97 if tok.token == tokenize2.NAME and not iskeyword(tok.text):
98 if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
98 if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
99 names[-1] = "%s.%s" % (names[-1], tok.text)
99 names[-1] = "%s.%s" % (names[-1], tok.text)
100 else:
100 else:
101 names.append(tok.text)
101 names.append(tok.text)
102 elif tok.token == tokenize2.OP:
102 elif tok.token == tokenize2.OP:
103 if tok.text == '=' and names:
103 if tok.text == '=' and names:
104 # don't inspect the lhs of an assignment
104 # don't inspect the lhs of an assignment
105 names.pop(-1)
105 names.pop(-1)
106 if tok.text == '(' and names:
106 if tok.text == '(' and names:
107 # if we are inside a function call, inspect the function
107 # if we are inside a function call, inspect the function
108 call_names.append(names[-1])
108 call_names.append(names[-1])
109 elif tok.text == ')' and call_names:
109 elif tok.text == ')' and call_names:
110 call_names.pop(-1)
110 call_names.pop(-1)
111
111
112 tokens.append(tok)
112 tokens.append(tok)
113
113
114 if offsets[end_line] + end_col > cursor_pos:
114 if offsets[end_line] + end_col > cursor_pos:
115 # we found the cursor, stop reading
115 # we found the cursor, stop reading
116 break
116 break
117
117
118 if call_names:
118 if call_names:
119 return call_names[-1]
119 return call_names[-1]
120 elif names:
120 elif names:
121 return names[-1]
121 return names[-1]
122 else:
122 else:
123 return ''
123 return ''
124
124
125
125
General Comments 0
You need to be logged in to leave comments. Login now