Show More
@@ -1274,8 +1274,8 def _tokenize(text): | |||||
1274 | yield (b'symbol', symbol, pos) |
|
1274 | yield (b'symbol', symbol, pos) | |
1275 | pos += len(symbol) |
|
1275 | pos += len(symbol) | |
1276 | else: # special char, ignore space |
|
1276 | else: # special char, ignore space | |
1277 | if text[pos] != b' ': |
|
1277 | if text[pos : pos + 1] != b' ': | |
1278 | yield (text[pos], None, pos) |
|
1278 | yield (text[pos : pos + 1], None, pos) | |
1279 | pos += 1 |
|
1279 | pos += 1 | |
1280 | yield (b'end', None, pos) |
|
1280 | yield (b'end', None, pos) | |
1281 |
|
1281 |
General Comments 0
You need to be logged in to leave comments.
Login now