Show More
@@ -45,6 +45,9 b' elements = {' | |||
|
45 | 45 | |
|
46 | 46 | keywords = set(['and', 'or', 'not']) |
|
47 | 47 | |
|
48 | _quoteletters = set(['"', "'"]) | |
|
49 | _simpleopletters = set(pycompat.iterbytestr("():=,-|&+!~^%")) | |
|
50 | ||
|
48 | 51 | # default set of valid characters for the initial letter of symbols |
|
49 | 52 | _syminitletters = set(pycompat.iterbytestr( |
|
50 | 53 | string.ascii_letters.encode('ascii') + |
@@ -109,9 +112,9 b' def tokenize(program, lookup=None, symin' | |||
|
109 | 112 | elif c == '#' and program[pos:pos + 2] == '##': # look ahead carefully |
|
110 | 113 | yield ('##', None, pos) |
|
111 | 114 | pos += 1 # skip ahead |
|
112 |
elif c in |
|
|
115 | elif c in _simpleopletters: # handle simple operators | |
|
113 | 116 | yield (c, None, pos) |
|
114 |
elif (c in |
|
|
117 | elif (c in _quoteletters or c == 'r' and | |
|
115 | 118 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings |
|
116 | 119 | if c == 'r': |
|
117 | 120 | pos += 1 |
General Comments 0
You need to be logged in to leave comments.
Login now