Show More
@@ -24,11 +24,12 b" def get_tokens_unprocessed(self, text, stack=('root',)):" | |||||
24 | for rexmatch, action, new_state in statetokens: |
|
24 | for rexmatch, action, new_state in statetokens: | |
25 | m = rexmatch(text, pos) |
|
25 | m = rexmatch(text, pos) | |
26 | if m: |
|
26 | if m: | |
27 |
if |
|
27 | if action is not None: | |
28 |
|
|
28 | if type(action) is _TokenType: | |
29 | else: |
|
29 | yield pos, action, m.group() | |
30 |
|
|
30 | else: | |
31 |
|
|
31 | for item in action(self, m): | |
|
32 | yield item | |||
32 | pos = m.end() |
|
33 | pos = m.end() | |
33 | if new_state is not None: |
|
34 | if new_state is not None: | |
34 | # state transition |
|
35 | # state transition |
General Comments 0
You need to be logged in to leave comments.
Login now