##// END OF EJS Templates
templater: remove processing of "string" literals from tokenizer...
Yuya Nishihara -
r25784:33e61368 default
parent child Browse files
Show More
@@ -22,7 +22,6 b' elements = {'
22 ")": (0, None, None),
22 ")": (0, None, None),
23 "integer": (0, ("integer",), None),
23 "integer": (0, ("integer",), None),
24 "symbol": (0, ("symbol",), None),
24 "symbol": (0, ("symbol",), None),
25 "string": (0, ("template",), None),
26 "rawstring": (0, ("rawstring",), None),
25 "rawstring": (0, ("rawstring",), None),
27 "template": (0, ("template",), None),
26 "template": (0, ("template",), None),
28 "end": (0, None, None),
27 "end": (0, None, None),
@@ -41,26 +40,17 b' def tokenize(program, start, end):'
41 data, pos = _parsetemplate(program, s, end, c)
40 data, pos = _parsetemplate(program, s, end, c)
42 yield ('template', data, s)
41 yield ('template', data, s)
43 pos -= 1
42 pos -= 1
44 elif (c in '"\'' or c == 'r' and
43 elif c == 'r' and program[pos:pos + 2] in ("r'", 'r"'):
45 program[pos:pos + 2] in ("r'", 'r"')): # handle quoted strings
44 # handle quoted strings
46 if c == 'r':
45 c = program[pos + 1]
47 pos += 1
46 s = pos = pos + 2
48 c = program[pos]
49 decode = False
50 else:
51 decode = True
52 pos += 1
53 s = pos
54 while pos < end: # find closing quote
47 while pos < end: # find closing quote
55 d = program[pos]
48 d = program[pos]
56 if d == '\\': # skip over escaped characters
49 if d == '\\': # skip over escaped characters
57 pos += 2
50 pos += 2
58 continue
51 continue
59 if d == c:
52 if d == c:
60 if not decode:
53 yield ('rawstring', program[s:pos], s)
61 yield ('rawstring', program[s:pos], s)
62 break
63 yield ('string', program[s:pos], s)
64 break
54 break
65 pos += 1
55 pos += 1
66 else:
56 else:
General Comments 0
You need to be logged in to leave comments. Login now