Show More
@@ -675,7 +675,7 b' def _tokenize(text):' | |||||
675 | length = len(text) |
|
675 | length = len(text) | |
676 | while pos < length: |
|
676 | while pos < length: | |
677 | symbol = b''.join(itertools.takewhile(lambda ch: ch not in special, |
|
677 | symbol = b''.join(itertools.takewhile(lambda ch: ch not in special, | |
678 | view[pos:])) |
|
678 | pycompat.iterbytestr(view[pos:]))) | |
679 | if symbol: |
|
679 | if symbol: | |
680 | yield (b'symbol', symbol, pos) |
|
680 | yield (b'symbol', symbol, pos) | |
681 | pos += len(symbol) |
|
681 | pos += len(symbol) |
General Comments 0
You need to be logged in to leave comments.
Login now