Show More
@@ -22,7 +22,6 b' elements = {' | |||||
22 | ")": (0, None, None), |
|
22 | ")": (0, None, None), | |
23 | "integer": (0, ("integer",), None), |
|
23 | "integer": (0, ("integer",), None), | |
24 | "symbol": (0, ("symbol",), None), |
|
24 | "symbol": (0, ("symbol",), None), | |
25 | "string": (0, ("template",), None), |
|
|||
26 | "rawstring": (0, ("rawstring",), None), |
|
25 | "rawstring": (0, ("rawstring",), None), | |
27 | "template": (0, ("template",), None), |
|
26 | "template": (0, ("template",), None), | |
28 | "end": (0, None, None), |
|
27 | "end": (0, None, None), | |
@@ -41,26 +40,17 b' def tokenize(program, start, end):' | |||||
41 | data, pos = _parsetemplate(program, s, end, c) |
|
40 | data, pos = _parsetemplate(program, s, end, c) | |
42 | yield ('template', data, s) |
|
41 | yield ('template', data, s) | |
43 | pos -= 1 |
|
42 | pos -= 1 | |
44 | elif (c in '"\'' or c == 'r' and |
|
43 | elif c == 'r' and program[pos:pos + 2] in ("r'", 'r"'): | |
45 |
|
|
44 | # handle quoted strings | |
46 |
|
|
45 | c = program[pos + 1] | |
47 |
pos + |
|
46 | s = pos = pos + 2 | |
48 | c = program[pos] |
|
|||
49 | decode = False |
|
|||
50 | else: |
|
|||
51 | decode = True |
|
|||
52 | pos += 1 |
|
|||
53 | s = pos |
|
|||
54 | while pos < end: # find closing quote |
|
47 | while pos < end: # find closing quote | |
55 | d = program[pos] |
|
48 | d = program[pos] | |
56 | if d == '\\': # skip over escaped characters |
|
49 | if d == '\\': # skip over escaped characters | |
57 | pos += 2 |
|
50 | pos += 2 | |
58 | continue |
|
51 | continue | |
59 | if d == c: |
|
52 | if d == c: | |
60 | if not decode: |
|
53 | yield ('rawstring', program[s:pos], s) | |
61 | yield ('rawstring', program[s:pos], s) |
|
|||
62 | break |
|
|||
63 | yield ('string', program[s:pos], s) |
|
|||
64 | break |
|
54 | break | |
65 | pos += 1 |
|
55 | pos += 1 | |
66 | else: |
|
56 | else: |
General Comments 0
You need to be logged in to leave comments.
Login now