Show More
@@ -36,18 +36,21 b' def tokenizer(data):' | |||||
36 | if c == 'r': |
|
36 | if c == 'r': | |
37 | pos += 1 |
|
37 | pos += 1 | |
38 | c = program[pos] |
|
38 | c = program[pos] | |
39 |
decode = |
|
39 | decode = False | |
40 | else: |
|
40 | else: | |
41 | decode = lambda x: x.decode('string-escape') |
|
41 | decode = True | |
42 | pos += 1 |
|
42 | pos += 1 | |
43 | s = pos |
|
43 | s = pos | |
44 | while pos < end: # find closing quote |
|
44 | while pos < end: # find closing quote | |
45 | d = program[pos] |
|
45 | d = program[pos] | |
46 | if d == '\\': # skip over escaped characters |
|
46 | if decode and d == '\\': # skip over escaped characters | |
47 | pos += 2 |
|
47 | pos += 2 | |
48 | continue |
|
48 | continue | |
49 | if d == c: |
|
49 | if d == c: | |
50 |
|
|
50 | if not decode: | |
|
51 | yield ('string', program[s:pos].replace('\\', r'\\'), s) | |||
|
52 | break | |||
|
53 | yield ('string', program[s:pos].decode('string-escape'), s) | |||
51 | break |
|
54 | break | |
52 | pos += 1 |
|
55 | pos += 1 | |
53 | else: |
|
56 | else: |
General Comments 0
You need to be logged in to leave comments.
Login now