Show More
@@ -178,7 +178,7 b' def tokenize(program, lookup=None):' | |||||
178 | pos += 1 |
|
178 | pos += 1 | |
179 | while pos < l: # find end of symbol |
|
179 | while pos < l: # find end of symbol | |
180 | d = program[pos] |
|
180 | d = program[pos] | |
181 | if not (d.isalnum() or d in "._/@" or ord(d) > 127): |
|
181 | if not (d.isalnum() or d in "-._/@" or ord(d) > 127): | |
182 | break |
|
182 | break | |
183 | if d == '.' and program[pos - 1] == '.': # special case for .. |
|
183 | if d == '.' and program[pos - 1] == '.': # special case for .. | |
184 | pos -= 1 |
|
184 | pos -= 1 | |
@@ -187,6 +187,22 b' def tokenize(program, lookup=None):' | |||||
187 | sym = program[s:pos] |
|
187 | sym = program[s:pos] | |
188 | if sym in keywords: # operator keywords |
|
188 | if sym in keywords: # operator keywords | |
189 | yield (sym, None, s) |
|
189 | yield (sym, None, s) | |
|
190 | elif '-' in sym: | |||
|
191 | # some jerk gave us foo-bar-baz, try to check if it's a symbol | |||
|
192 | if lookup and lookup(sym): | |||
|
193 | # looks like a real symbol | |||
|
194 | yield ('symbol', sym, s) | |||
|
195 | else: | |||
|
196 | # looks like an expression | |||
|
197 | parts = sym.split('-') | |||
|
198 | for p in parts[:-1]: | |||
|
199 | if p: # possible consecutive - | |||
|
200 | yield ('symbol', p, s) | |||
|
201 | s += len(p) | |||
|
202 | yield ('-', None, pos) | |||
|
203 | s += 1 | |||
|
204 | if parts[-1]: # possible trailing - | |||
|
205 | yield ('symbol', parts[-1], s) | |||
190 | else: |
|
206 | else: | |
191 | yield ('symbol', sym, s) |
|
207 | yield ('symbol', sym, s) | |
192 | pos -= 1 |
|
208 | pos -= 1 |
General Comments 0
You need to be logged in to leave comments.
Login now