Show More
@@ -45,6 +45,9 b' elements = {' | |||||
45 |
|
45 | |||
46 | keywords = set(['and', 'or', 'not']) |
|
46 | keywords = set(['and', 'or', 'not']) | |
47 |
|
47 | |||
|
48 | _quoteletters = set(['"', "'"]) | |||
|
49 | _simpleopletters = set(pycompat.iterbytestr("():=,-|&+!~^%")) | |||
|
50 | ||||
48 | # default set of valid characters for the initial letter of symbols |
|
51 | # default set of valid characters for the initial letter of symbols | |
49 | _syminitletters = set(pycompat.iterbytestr( |
|
52 | _syminitletters = set(pycompat.iterbytestr( | |
50 | string.ascii_letters.encode('ascii') + |
|
53 | string.ascii_letters.encode('ascii') + | |
@@ -109,9 +112,9 b' def tokenize(program, lookup=None, symin' | |||||
109 | elif c == '#' and program[pos:pos + 2] == '##': # look ahead carefully |
|
112 | elif c == '#' and program[pos:pos + 2] == '##': # look ahead carefully | |
110 | yield ('##', None, pos) |
|
113 | yield ('##', None, pos) | |
111 | pos += 1 # skip ahead |
|
114 | pos += 1 # skip ahead | |
112 |
elif c in |
|
115 | elif c in _simpleopletters: # handle simple operators | |
113 | yield (c, None, pos) |
|
116 | yield (c, None, pos) | |
114 |
elif (c in |
|
117 | elif (c in _quoteletters or c == 'r' and | |
115 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings |
|
118 | program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings | |
116 | if c == 'r': |
|
119 | if c == 'r': | |
117 | pos += 1 |
|
120 | pos += 1 |
General Comments 0
You need to be logged in to leave comments.
Login now