Show More
@@ -1,129 +1,176 | |||||
1 | """Test lexers module""" |
|
1 | """Test lexers module""" | |
2 |
|
2 | |||
3 | # Copyright (c) IPython Development Team. |
|
3 | # Copyright (c) IPython Development Team. | |
4 | # Distributed under the terms of the Modified BSD License. |
|
4 | # Distributed under the terms of the Modified BSD License. | |
5 |
|
5 | |||
6 | from unittest import TestCase |
|
6 | from unittest import TestCase | |
7 | from pygments.token import Token |
|
7 | from pygments.token import Token | |
8 | from pygments.lexers import BashLexer |
|
8 | from pygments.lexers import BashLexer | |
9 |
|
9 | |||
10 | from .. import lexers |
|
10 | from .. import lexers | |
11 |
|
11 | |||
12 |
|
12 | |||
13 | class TestLexers(TestCase): |
|
13 | class TestLexers(TestCase): | |
14 | """Collection of lexers tests""" |
|
14 | """Collection of lexers tests""" | |
15 | def setUp(self): |
|
15 | def setUp(self): | |
16 | self.lexer = lexers.IPythonLexer() |
|
16 | self.lexer = lexers.IPythonLexer() | |
17 | self.bash_lexer = BashLexer() |
|
17 | self.bash_lexer = BashLexer() | |
18 |
|
18 | |||
19 | def testIPythonLexer(self): |
|
19 | def testIPythonLexer(self): | |
20 | fragment = '!echo $HOME\n' |
|
20 | fragment = '!echo $HOME\n' | |
21 | tokens = [ |
|
21 | tokens = [ | |
22 | (Token.Operator, '!'), |
|
22 | (Token.Operator, '!'), | |
23 | ] |
|
23 | ] | |
24 | tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) |
|
24 | tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) | |
25 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) |
|
25 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) | |
26 |
|
26 | |||
27 | fragment_2 = '!' + fragment |
|
27 | fragment_2 = '!' + fragment | |
28 | tokens_2 = [ |
|
28 | tokens_2 = [ | |
29 | (Token.Operator, '!!'), |
|
29 | (Token.Operator, '!!'), | |
30 | ] + tokens[1:] |
|
30 | ] + tokens[1:] | |
31 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
31 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
32 |
|
32 | |||
33 | fragment_2 = '\t %%!\n' + fragment[1:] |
|
33 | fragment_2 = '\t %%!\n' + fragment[1:] | |
34 | tokens_2 = [ |
|
34 | tokens_2 = [ | |
35 | (Token.Text, '\t '), |
|
35 | (Token.Text, '\t '), | |
36 | (Token.Operator, '%%!'), |
|
36 | (Token.Operator, '%%!'), | |
37 | (Token.Text, '\n'), |
|
37 | (Token.Text, '\n'), | |
38 | ] + tokens[1:] |
|
38 | ] + tokens[1:] | |
39 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
39 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
40 |
|
40 | |||
41 | fragment_2 = 'x = ' + fragment |
|
41 | fragment_2 = 'x = ' + fragment | |
42 | tokens_2 = [ |
|
42 | tokens_2 = [ | |
43 | (Token.Name, 'x'), |
|
43 | (Token.Name, 'x'), | |
44 | (Token.Text, ' '), |
|
44 | (Token.Text, ' '), | |
45 | (Token.Operator, '='), |
|
45 | (Token.Operator, '='), | |
46 | (Token.Text, ' '), |
|
46 | (Token.Text, ' '), | |
47 | ] + tokens |
|
47 | ] + tokens | |
48 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
48 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
49 |
|
49 | |||
50 | fragment_2 = 'x, = ' + fragment |
|
50 | fragment_2 = 'x, = ' + fragment | |
51 | tokens_2 = [ |
|
51 | tokens_2 = [ | |
52 | (Token.Name, 'x'), |
|
52 | (Token.Name, 'x'), | |
53 | (Token.Punctuation, ','), |
|
53 | (Token.Punctuation, ','), | |
54 | (Token.Text, ' '), |
|
54 | (Token.Text, ' '), | |
55 | (Token.Operator, '='), |
|
55 | (Token.Operator, '='), | |
56 | (Token.Text, ' '), |
|
56 | (Token.Text, ' '), | |
57 | ] + tokens |
|
57 | ] + tokens | |
58 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
58 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
59 |
|
59 | |||
60 | fragment_2 = 'x, = %sx ' + fragment[1:] |
|
60 | fragment_2 = 'x, = %sx ' + fragment[1:] | |
61 | tokens_2 = [ |
|
61 | tokens_2 = [ | |
62 | (Token.Name, 'x'), |
|
62 | (Token.Name, 'x'), | |
63 | (Token.Punctuation, ','), |
|
63 | (Token.Punctuation, ','), | |
64 | (Token.Text, ' '), |
|
64 | (Token.Text, ' '), | |
65 | (Token.Operator, '='), |
|
65 | (Token.Operator, '='), | |
66 | (Token.Text, ' '), |
|
66 | (Token.Text, ' '), | |
67 | (Token.Operator, '%'), |
|
67 | (Token.Operator, '%'), | |
68 | (Token.Keyword, 'sx'), |
|
68 | (Token.Keyword, 'sx'), | |
69 | (Token.Text, ' '), |
|
69 | (Token.Text, ' '), | |
70 | ] + tokens[1:] |
|
70 | ] + tokens[1:] | |
71 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
71 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
72 |
|
72 | |||
73 | fragment_2 = 'f = %R function () {}\n' |
|
73 | fragment_2 = 'f = %R function () {}\n' | |
74 | tokens_2 = [ |
|
74 | tokens_2 = [ | |
75 | (Token.Name, 'f'), |
|
75 | (Token.Name, 'f'), | |
76 | (Token.Text, ' '), |
|
76 | (Token.Text, ' '), | |
77 | (Token.Operator, '='), |
|
77 | (Token.Operator, '='), | |
78 | (Token.Text, ' '), |
|
78 | (Token.Text, ' '), | |
79 | (Token.Operator, '%'), |
|
79 | (Token.Operator, '%'), | |
80 | (Token.Keyword, 'R'), |
|
80 | (Token.Keyword, 'R'), | |
81 | (Token.Text, ' function () {}\n'), |
|
81 | (Token.Text, ' function () {}\n'), | |
82 | ] |
|
82 | ] | |
83 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
83 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
84 |
|
84 | |||
85 | fragment_2 = '\t%%xyz\n$foo\n' |
|
85 | fragment_2 = '\t%%xyz\n$foo\n' | |
86 | tokens_2 = [ |
|
86 | tokens_2 = [ | |
87 | (Token.Text, '\t'), |
|
87 | (Token.Text, '\t'), | |
88 | (Token.Operator, '%%'), |
|
88 | (Token.Operator, '%%'), | |
89 | (Token.Keyword, 'xyz'), |
|
89 | (Token.Keyword, 'xyz'), | |
90 | (Token.Text, '\n$foo\n'), |
|
90 | (Token.Text, '\n$foo\n'), | |
91 | ] |
|
91 | ] | |
92 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
92 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
93 |
|
93 | |||
94 | fragment_2 = '%system?\n' |
|
94 | fragment_2 = '%system?\n' | |
95 | tokens_2 = [ |
|
95 | tokens_2 = [ | |
96 | (Token.Operator, '%'), |
|
96 | (Token.Operator, '%'), | |
97 | (Token.Keyword, 'system'), |
|
97 | (Token.Keyword, 'system'), | |
98 | (Token.Operator, '?'), |
|
98 | (Token.Operator, '?'), | |
99 | (Token.Text, '\n'), |
|
99 | (Token.Text, '\n'), | |
100 | ] |
|
100 | ] | |
101 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
101 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
102 |
|
102 | |||
103 | fragment_2 = 'x != y\n' |
|
103 | fragment_2 = 'x != y\n' | |
104 | tokens_2 = [ |
|
104 | tokens_2 = [ | |
105 | (Token.Name, 'x'), |
|
105 | (Token.Name, 'x'), | |
106 | (Token.Text, ' '), |
|
106 | (Token.Text, ' '), | |
107 | (Token.Operator, '!='), |
|
107 | (Token.Operator, '!='), | |
108 | (Token.Text, ' '), |
|
108 | (Token.Text, ' '), | |
109 | (Token.Name, 'y'), |
|
109 | (Token.Name, 'y'), | |
110 | (Token.Text, '\n'), |
|
110 | (Token.Text, '\n'), | |
111 | ] |
|
111 | ] | |
112 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
112 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
113 |
|
113 | |||
114 | fragment_2 = ' ?math.sin\n' |
|
114 | fragment_2 = ' ?math.sin\n' | |
115 | tokens_2 = [ |
|
115 | tokens_2 = [ | |
116 | (Token.Text, ' '), |
|
116 | (Token.Text, ' '), | |
117 | (Token.Operator, '?'), |
|
117 | (Token.Operator, '?'), | |
118 | (Token.Text, 'math.sin'), |
|
118 | (Token.Text, 'math.sin'), | |
119 | (Token.Text, '\n'), |
|
119 | (Token.Text, '\n'), | |
120 | ] |
|
120 | ] | |
121 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) |
|
121 | self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2))) | |
122 |
|
122 | |||
123 | fragment = ' *int*?\n' |
|
123 | fragment = ' *int*?\n' | |
124 | tokens = [ |
|
124 | tokens = [ | |
125 | (Token.Text, ' *int*'), |
|
125 | (Token.Text, ' *int*'), | |
126 | (Token.Operator, '?'), |
|
126 | (Token.Operator, '?'), | |
127 | (Token.Text, '\n'), |
|
127 | (Token.Text, '\n'), | |
128 | ] |
|
128 | ] | |
129 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) |
|
129 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) | |
|
130 | ||||
|
131 | fragment = '%%writefile -a foo.py\nif a == b:\n pass' | |||
|
132 | tokens = [ | |||
|
133 | (Token.Operator, '%%writefile'), | |||
|
134 | (Token.Text, ' -a foo.py\n'), | |||
|
135 | (Token.Keyword, 'if'), | |||
|
136 | (Token.Text, ' '), | |||
|
137 | (Token.Name, 'a'), | |||
|
138 | (Token.Text, ' '), | |||
|
139 | (Token.Operator, '=='), | |||
|
140 | (Token.Text, ' '), | |||
|
141 | (Token.Name, 'b'), | |||
|
142 | (Token.Punctuation, ':'), | |||
|
143 | (Token.Text, '\n'), | |||
|
144 | (Token.Text, ' '), | |||
|
145 | (Token.Keyword, 'pass'), | |||
|
146 | (Token.Text, '\n'), | |||
|
147 | ] | |||
|
148 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) | |||
|
149 | ||||
|
150 | fragment = '%%timeit\nmath.sin(0)' | |||
|
151 | tokens = [ | |||
|
152 | (Token.Operator, '%%timeit\n'), | |||
|
153 | (Token.Name, 'math'), | |||
|
154 | (Token.Operator, '.'), | |||
|
155 | (Token.Name, 'sin'), | |||
|
156 | (Token.Punctuation, '('), | |||
|
157 | (Token.Literal.Number.Integer, '0'), | |||
|
158 | (Token.Punctuation, ')'), | |||
|
159 | (Token.Text, '\n'), | |||
|
160 | ] | |||
|
161 | ||||
|
162 | fragment = '%%HTML\n<div>foo</div>' | |||
|
163 | tokens = [ | |||
|
164 | (Token.Operator, '%%HTML'), | |||
|
165 | (Token.Text, '\n'), | |||
|
166 | (Token.Punctuation, '<'), | |||
|
167 | (Token.Name.Tag, 'div'), | |||
|
168 | (Token.Punctuation, '>'), | |||
|
169 | (Token.Text, 'foo'), | |||
|
170 | (Token.Punctuation, '<'), | |||
|
171 | (Token.Punctuation, '/'), | |||
|
172 | (Token.Name.Tag, 'div'), | |||
|
173 | (Token.Punctuation, '>'), | |||
|
174 | (Token.Text, '\n'), | |||
|
175 | ] | |||
|
176 | self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) |
General Comments 0
You need to be logged in to leave comments.
Login now