Show More
@@ -177,7 +177,7 b' def tokenize(program, lookup=None, symin' | |||||
177 | if p: # possible consecutive - |
|
177 | if p: # possible consecutive - | |
178 | yield ('symbol', p, s) |
|
178 | yield ('symbol', p, s) | |
179 | s += len(p) |
|
179 | s += len(p) | |
180 |
yield ('-', None, |
|
180 | yield ('-', None, s) | |
181 | s += 1 |
|
181 | s += 1 | |
182 | if parts[-1]: # possible trailing - |
|
182 | if parts[-1]: # possible trailing - | |
183 | yield ('symbol', parts[-1], s) |
|
183 | yield ('symbol', parts[-1], s) |
General Comments 0
You need to be logged in to leave comments.
Login now