Show More
@@ -96,7 +96,7 b' def tokenize(program, lookup=None, symin' | |||||
96 |
|
96 | |||
97 | pos, l = 0, len(program) |
|
97 | pos, l = 0, len(program) | |
98 | while pos < l: |
|
98 | while pos < l: | |
99 | c = program[pos] |
|
99 | c = program[pos:pos + 1] | |
100 | if c.isspace(): # skip inter-token whitespace |
|
100 | if c.isspace(): # skip inter-token whitespace | |
101 | pass |
|
101 | pass | |
102 | elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully |
|
102 | elif c == ':' and program[pos:pos + 2] == '::': # look ahead carefully | |
@@ -114,14 +114,14 b' def tokenize(program, lookup=None, symin' | |||||
114 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings |
|
114 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings | |
115 | if c == 'r': |
|
115 | if c == 'r': | |
116 | pos += 1 |
|
116 | pos += 1 | |
117 | c = program[pos] |
|
117 | c = program[pos:pos + 1] | |
118 | decode = lambda x: x |
|
118 | decode = lambda x: x | |
119 | else: |
|
119 | else: | |
120 | decode = parser.unescapestr |
|
120 | decode = parser.unescapestr | |
121 | pos += 1 |
|
121 | pos += 1 | |
122 | s = pos |
|
122 | s = pos | |
123 | while pos < l: # find closing quote |
|
123 | while pos < l: # find closing quote | |
124 | d = program[pos] |
|
124 | d = program[pos:pos + 1] | |
125 | if d == '\\': # skip over escaped characters |
|
125 | if d == '\\': # skip over escaped characters | |
126 | pos += 2 |
|
126 | pos += 2 | |
127 | continue |
|
127 | continue | |
@@ -136,10 +136,11 b' def tokenize(program, lookup=None, symin' | |||||
136 | s = pos |
|
136 | s = pos | |
137 | pos += 1 |
|
137 | pos += 1 | |
138 | while pos < l: # find end of symbol |
|
138 | while pos < l: # find end of symbol | |
139 | d = program[pos] |
|
139 | d = program[pos:pos + 1] | |
140 | if d not in symletters: |
|
140 | if d not in symletters: | |
141 | break |
|
141 | break | |
142 | if d == '.' and program[pos - 1] == '.': # special case for .. |
|
142 | if (d == '.' | |
|
143 | and program[pos - 1:pos] == '.'): # special case for .. | |||
143 | pos -= 1 |
|
144 | pos -= 1 | |
144 | break |
|
145 | break | |
145 | pos += 1 |
|
146 | pos += 1 |
General Comments 0
You need to be logged in to leave comments.
Login now