Show More
@@ -96,7 +96,7 b' def tokenize(program, lookup=None, symin' | |||
|
96 | 96 | |
|
97 | 97 | pos, l = 0, len(program) |
|
98 | 98 | while pos < l: |
|
99 | c = program[pos] | |
|
99 | c = program[pos:pos + 1] | |
|
100 | 100 | if c.isspace(): # skip inter-token whitespace |
|
101 | 101 | pass |
|
102 | 102 | elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully |
@@ -114,14 +114,14 b' def tokenize(program, lookup=None, symin' | |||
|
114 | 114 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings |
|
115 | 115 | if c == 'r': |
|
116 | 116 | pos += 1 |
|
117 | c = program[pos] | |
|
117 | c = program[pos:pos + 1] | |
|
118 | 118 | decode = lambda x: x |
|
119 | 119 | else: |
|
120 | 120 | decode = parser.unescapestr |
|
121 | 121 | pos += 1 |
|
122 | 122 | s = pos |
|
123 | 123 | while pos < l: # find closing quote |
|
124 | d = program[pos] | |
|
124 | d = program[pos:pos + 1] | |
|
125 | 125 | if d == '\\': # skip over escaped characters |
|
126 | 126 | pos += 2 |
|
127 | 127 | continue |
@@ -136,10 +136,11 b' def tokenize(program, lookup=None, symin' | |||
|
136 | 136 | s = pos |
|
137 | 137 | pos += 1 |
|
138 | 138 | while pos < l: # find end of symbol |
|
139 | d = program[pos] | |
|
139 | d = program[pos:pos + 1] | |
|
140 | 140 | if d not in symletters: |
|
141 | 141 | break |
|
142 | if d == '.' and program[pos - 1] == '.': # special case for .. | |
|
142 | if (d == '.' | |
|
143 | and program[pos - 1:pos] == '.'): # special case for .. | |
|
143 | 144 | pos -= 1 |
|
144 | 145 | break |
|
145 | 146 | pos += 1 |
General Comments 0
You need to be logged in to leave comments.
Login now