Show More
@@ -178,7 +178,7 b' def tokenize(program, lookup=None):' | |||
|
178 | 178 | pos += 1 |
|
179 | 179 | while pos < l: # find end of symbol |
|
180 | 180 | d = program[pos] |
|
181 | if not (d.isalnum() or d in "._/@" or ord(d) > 127): | |
|
181 | if not (d.isalnum() or d in "-._/@" or ord(d) > 127): | |
|
182 | 182 | break |
|
183 | 183 | if d == '.' and program[pos - 1] == '.': # special case for .. |
|
184 | 184 | pos -= 1 |
@@ -187,6 +187,22 b' def tokenize(program, lookup=None):' | |||
|
187 | 187 | sym = program[s:pos] |
|
188 | 188 | if sym in keywords: # operator keywords |
|
189 | 189 | yield (sym, None, s) |
|
190 | elif '-' in sym: | |
|
191 | # some jerk gave us foo-bar-baz, try to check if it's a symbol | |
|
192 | if lookup and lookup(sym): | |
|
193 | # looks like a real symbol | |
|
194 | yield ('symbol', sym, s) | |
|
195 | else: | |
|
196 | # looks like an expression | |
|
197 | parts = sym.split('-') | |
|
198 | for p in parts[:-1]: | |
|
199 | if p: # possible consecutive - | |
|
200 | yield ('symbol', p, s) | |
|
201 | s += len(p) | |
|
202 | yield ('-', None, pos) | |
|
203 | s += 1 | |
|
204 | if parts[-1]: # possible trailing - | |
|
205 | yield ('symbol', parts[-1], s) | |
|
190 | 206 | else: |
|
191 | 207 | yield ('symbol', sym, s) |
|
192 | 208 | pos -= 1 |
General Comments 0
You need to be logged in to leave comments.
Login now