Show More
@@ -675,7 +675,7 b' def _tokenize(text):' | |||
|
675 | 675 | length = len(text) |
|
676 | 676 | while pos < length: |
|
677 | 677 | symbol = b''.join(itertools.takewhile(lambda ch: ch not in special, |
|
678 | view[pos:])) | |
|
678 | pycompat.iterbytestr(view[pos:]))) | |
|
679 | 679 | if symbol: |
|
680 | 680 | yield (b'symbol', symbol, pos) |
|
681 | 681 | pos += len(symbol) |
General Comments 0
You need to be logged in to leave comments.
Login now