Show More
@@ -1274,8 +1274,8 b' def _tokenize(text):' | |||
|
1274 | 1274 | yield (b'symbol', symbol, pos) |
|
1275 | 1275 | pos += len(symbol) |
|
1276 | 1276 | else: # special char, ignore space |
|
1277 | if text[pos] != b' ': | |
|
1278 | yield (text[pos], None, pos) | |
|
1277 | if text[pos : pos + 1] != b' ': | |
|
1278 | yield (text[pos : pos + 1], None, pos) | |
|
1279 | 1279 | pos += 1 |
|
1280 | 1280 | yield (b'end', None, pos) |
|
1281 | 1281 |
General Comments 0
You need to be logged in to leave comments.
Login now