Show More
@@ -1,299 +1,303 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | Class and program to colorize python source code for ANSI terminals. |
|
3 | Class and program to colorize python source code for ANSI terminals. | |
4 |
|
4 | |||
5 | Based on an HTML code highlighter by Jurgen Hermann found at: |
|
5 | Based on an HTML code highlighter by Jurgen Hermann found at: | |
6 | http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298 |
|
6 | http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298 | |
7 |
|
7 | |||
8 | Modifications by Fernando Perez (fperez@colorado.edu). |
|
8 | Modifications by Fernando Perez (fperez@colorado.edu). | |
9 |
|
9 | |||
10 | Information on the original HTML highlighter follows: |
|
10 | Information on the original HTML highlighter follows: | |
11 |
|
11 | |||
12 | MoinMoin - Python Source Parser |
|
12 | MoinMoin - Python Source Parser | |
13 |
|
13 | |||
14 | Title: Colorize Python source using the built-in tokenizer |
|
14 | Title: Colorize Python source using the built-in tokenizer | |
15 |
|
15 | |||
16 | Submitter: Jurgen Hermann |
|
16 | Submitter: Jurgen Hermann | |
17 | Last Updated:2001/04/06 |
|
17 | Last Updated:2001/04/06 | |
18 |
|
18 | |||
19 | Version no:1.2 |
|
19 | Version no:1.2 | |
20 |
|
20 | |||
21 | Description: |
|
21 | Description: | |
22 |
|
22 | |||
23 | This code is part of MoinMoin (http://moin.sourceforge.net/) and converts |
|
23 | This code is part of MoinMoin (http://moin.sourceforge.net/) and converts | |
24 | Python source code to HTML markup, rendering comments, keywords, |
|
24 | Python source code to HTML markup, rendering comments, keywords, | |
25 | operators, numeric and string literals in different colors. |
|
25 | operators, numeric and string literals in different colors. | |
26 |
|
26 | |||
27 | It shows how to use the built-in keyword, token and tokenize modules to |
|
27 | It shows how to use the built-in keyword, token and tokenize modules to | |
28 | scan Python source code and re-emit it with no changes to its original |
|
28 | scan Python source code and re-emit it with no changes to its original | |
29 | formatting (which is the hard part). |
|
29 | formatting (which is the hard part). | |
30 | """ |
|
30 | """ | |
31 |
|
31 | |||
32 | __all__ = ['ANSICodeColors','Parser'] |
|
32 | __all__ = ['ANSICodeColors','Parser'] | |
33 |
|
33 | |||
34 | _scheme_default = 'Linux' |
|
34 | _scheme_default = 'Linux' | |
35 |
|
35 | |||
36 | # Imports |
|
36 | # Imports | |
37 | import cStringIO |
|
37 | import cStringIO | |
38 | import keyword |
|
38 | import keyword | |
39 | import os |
|
39 | import os | |
40 | import optparse |
|
40 | import optparse | |
41 | import sys |
|
41 | import sys | |
42 | import token |
|
42 | import token | |
43 | import tokenize |
|
43 | import tokenize | |
44 |
|
44 | |||
45 | from IPython.utils.coloransi import * |
|
45 | from IPython.utils.coloransi import * | |
46 |
|
46 | |||
47 | ############################################################################# |
|
47 | ############################################################################# | |
48 | ### Python Source Parser (does Hilighting) |
|
48 | ### Python Source Parser (does Hilighting) | |
49 | ############################################################################# |
|
49 | ############################################################################# | |
50 |
|
50 | |||
51 | _KEYWORD = token.NT_OFFSET + 1 |
|
51 | _KEYWORD = token.NT_OFFSET + 1 | |
52 | _TEXT = token.NT_OFFSET + 2 |
|
52 | _TEXT = token.NT_OFFSET + 2 | |
53 |
|
53 | |||
54 | #**************************************************************************** |
|
54 | #**************************************************************************** | |
55 | # Builtin color schemes |
|
55 | # Builtin color schemes | |
56 |
|
56 | |||
57 | Colors = TermColors # just a shorthand |
|
57 | Colors = TermColors # just a shorthand | |
58 |
|
58 | |||
59 | # Build a few color schemes |
|
59 | # Build a few color schemes | |
60 | NoColor = ColorScheme( |
|
60 | NoColor = ColorScheme( | |
61 | 'NoColor',{ |
|
61 | 'NoColor',{ | |
62 | token.NUMBER : Colors.NoColor, |
|
62 | token.NUMBER : Colors.NoColor, | |
63 | token.OP : Colors.NoColor, |
|
63 | token.OP : Colors.NoColor, | |
64 | token.STRING : Colors.NoColor, |
|
64 | token.STRING : Colors.NoColor, | |
65 | tokenize.COMMENT : Colors.NoColor, |
|
65 | tokenize.COMMENT : Colors.NoColor, | |
66 | token.NAME : Colors.NoColor, |
|
66 | token.NAME : Colors.NoColor, | |
67 | token.ERRORTOKEN : Colors.NoColor, |
|
67 | token.ERRORTOKEN : Colors.NoColor, | |
68 |
|
68 | |||
69 | _KEYWORD : Colors.NoColor, |
|
69 | _KEYWORD : Colors.NoColor, | |
70 | _TEXT : Colors.NoColor, |
|
70 | _TEXT : Colors.NoColor, | |
71 |
|
71 | |||
72 | 'normal' : Colors.NoColor # color off (usu. Colors.Normal) |
|
72 | 'normal' : Colors.NoColor # color off (usu. Colors.Normal) | |
73 | } ) |
|
73 | } ) | |
74 |
|
74 | |||
75 | LinuxColors = ColorScheme( |
|
75 | LinuxColors = ColorScheme( | |
76 | 'Linux',{ |
|
76 | 'Linux',{ | |
77 | token.NUMBER : Colors.LightCyan, |
|
77 | token.NUMBER : Colors.LightCyan, | |
78 | token.OP : Colors.Yellow, |
|
78 | token.OP : Colors.Yellow, | |
79 | token.STRING : Colors.LightBlue, |
|
79 | token.STRING : Colors.LightBlue, | |
80 | tokenize.COMMENT : Colors.LightRed, |
|
80 | tokenize.COMMENT : Colors.LightRed, | |
81 | token.NAME : Colors.White, |
|
81 | token.NAME : Colors.White, | |
82 | token.ERRORTOKEN : Colors.Red, |
|
82 | token.ERRORTOKEN : Colors.Red, | |
83 |
|
83 | |||
84 | _KEYWORD : Colors.LightGreen, |
|
84 | _KEYWORD : Colors.LightGreen, | |
85 | _TEXT : Colors.Yellow, |
|
85 | _TEXT : Colors.Yellow, | |
86 |
|
86 | |||
87 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) |
|
87 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) | |
88 | } ) |
|
88 | } ) | |
89 |
|
89 | |||
90 | LightBGColors = ColorScheme( |
|
90 | LightBGColors = ColorScheme( | |
91 | 'LightBG',{ |
|
91 | 'LightBG',{ | |
92 | token.NUMBER : Colors.Cyan, |
|
92 | token.NUMBER : Colors.Cyan, | |
93 | token.OP : Colors.Blue, |
|
93 | token.OP : Colors.Blue, | |
94 | token.STRING : Colors.Blue, |
|
94 | token.STRING : Colors.Blue, | |
95 | tokenize.COMMENT : Colors.Red, |
|
95 | tokenize.COMMENT : Colors.Red, | |
96 | token.NAME : Colors.Black, |
|
96 | token.NAME : Colors.Black, | |
97 | token.ERRORTOKEN : Colors.Red, |
|
97 | token.ERRORTOKEN : Colors.Red, | |
98 |
|
98 | |||
99 | _KEYWORD : Colors.Green, |
|
99 | _KEYWORD : Colors.Green, | |
100 | _TEXT : Colors.Blue, |
|
100 | _TEXT : Colors.Blue, | |
101 |
|
101 | |||
102 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) |
|
102 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) | |
103 | } ) |
|
103 | } ) | |
104 |
|
104 | |||
105 | # Build table of color schemes (needed by the parser) |
|
105 | # Build table of color schemes (needed by the parser) | |
106 | ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors], |
|
106 | ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors], | |
107 | _scheme_default) |
|
107 | _scheme_default) | |
108 |
|
108 | |||
109 | class Parser: |
|
109 | class Parser: | |
110 | """ Format colored Python source. |
|
110 | """ Format colored Python source. | |
111 | """ |
|
111 | """ | |
112 |
|
112 | |||
113 | def __init__(self, color_table=None,out = sys.stdout): |
|
113 | def __init__(self, color_table=None,out = sys.stdout): | |
114 | """ Create a parser with a specified color table and output channel. |
|
114 | """ Create a parser with a specified color table and output channel. | |
115 |
|
115 | |||
116 | Call format() to process code. |
|
116 | Call format() to process code. | |
117 | """ |
|
117 | """ | |
118 | self.color_table = color_table and color_table or ANSICodeColors |
|
118 | self.color_table = color_table and color_table or ANSICodeColors | |
119 | self.out = out |
|
119 | self.out = out | |
120 |
|
120 | |||
121 | def format(self, raw, out = None, scheme = ''): |
|
121 | def format(self, raw, out = None, scheme = ''): | |
122 | return self.format2(raw, out, scheme)[0] |
|
122 | return self.format2(raw, out, scheme)[0] | |
123 |
|
123 | |||
124 | def format2(self, raw, out = None, scheme = ''): |
|
124 | def format2(self, raw, out = None, scheme = ''): | |
125 | """ Parse and send the colored source. |
|
125 | """ Parse and send the colored source. | |
126 |
|
126 | |||
127 | If out and scheme are not specified, the defaults (given to |
|
127 | If out and scheme are not specified, the defaults (given to | |
128 | constructor) are used. |
|
128 | constructor) are used. | |
129 |
|
129 | |||
130 | out should be a file-type object. Optionally, out can be given as the |
|
130 | out should be a file-type object. Optionally, out can be given as the | |
131 | string 'str' and the parser will automatically return the output in a |
|
131 | string 'str' and the parser will automatically return the output in a | |
132 | string.""" |
|
132 | string.""" | |
133 |
|
133 | |||
134 | string_output = 0 |
|
134 | string_output = 0 | |
135 | if out == 'str' or self.out == 'str' or \ |
|
135 | if out == 'str' or self.out == 'str' or \ | |
136 | isinstance(self.out,cStringIO.OutputType): |
|
136 | isinstance(self.out,cStringIO.OutputType): | |
137 | # XXX - I don't really like this state handling logic, but at this |
|
137 | # XXX - I don't really like this state handling logic, but at this | |
138 | # point I don't want to make major changes, so adding the |
|
138 | # point I don't want to make major changes, so adding the | |
139 | # isinstance() check is the simplest I can do to ensure correct |
|
139 | # isinstance() check is the simplest I can do to ensure correct | |
140 | # behavior. |
|
140 | # behavior. | |
141 | out_old = self.out |
|
141 | out_old = self.out | |
142 | self.out = cStringIO.StringIO() |
|
142 | self.out = cStringIO.StringIO() | |
143 | string_output = 1 |
|
143 | string_output = 1 | |
144 | elif out is not None: |
|
144 | elif out is not None: | |
145 | self.out = out |
|
145 | self.out = out | |
146 |
|
146 | |||
147 | # Fast return of the unmodified input for NoColor scheme |
|
147 | # Fast return of the unmodified input for NoColor scheme | |
148 | if scheme == 'NoColor': |
|
148 | if scheme == 'NoColor': | |
149 | error = False |
|
149 | error = False | |
150 | self.out.write(raw) |
|
150 | self.out.write(raw) | |
151 | if string_output: |
|
151 | if string_output: | |
152 | return raw,error |
|
152 | return raw,error | |
153 | else: |
|
153 | else: | |
154 | return None,error |
|
154 | return None,error | |
155 |
|
155 | |||
156 | # local shorthands |
|
156 | # local shorthands | |
157 | colors = self.color_table[scheme].colors |
|
157 | colors = self.color_table[scheme].colors | |
158 | self.colors = colors # put in object so __call__ sees it |
|
158 | self.colors = colors # put in object so __call__ sees it | |
159 |
|
159 | |||
160 | # Remove trailing whitespace and normalize tabs |
|
160 | # Remove trailing whitespace and normalize tabs | |
161 | self.raw = raw.expandtabs().rstrip() |
|
161 | self.raw = raw.expandtabs().rstrip() | |
162 |
|
162 | |||
163 | # store line offsets in self.lines |
|
163 | # store line offsets in self.lines | |
164 | self.lines = [0, 0] |
|
164 | self.lines = [0, 0] | |
165 | pos = 0 |
|
165 | pos = 0 | |
166 | raw_find = self.raw.find |
|
166 | raw_find = self.raw.find | |
167 | lines_append = self.lines.append |
|
167 | lines_append = self.lines.append | |
168 | while 1: |
|
168 | while 1: | |
169 | pos = raw_find('\n', pos) + 1 |
|
169 | pos = raw_find('\n', pos) + 1 | |
170 | if not pos: break |
|
170 | if not pos: break | |
171 | lines_append(pos) |
|
171 | lines_append(pos) | |
172 | lines_append(len(self.raw)) |
|
172 | lines_append(len(self.raw)) | |
173 |
|
173 | |||
174 | # parse the source and write it |
|
174 | # parse the source and write it | |
175 | self.pos = 0 |
|
175 | self.pos = 0 | |
176 | text = cStringIO.StringIO(self.raw) |
|
176 | text = cStringIO.StringIO(self.raw) | |
177 |
|
177 | |||
178 | error = False |
|
178 | error = False | |
179 | try: |
|
179 | try: | |
180 | tokenize.tokenize(text.readline, self) |
|
180 | tokenize.tokenize(text.readline, self) | |
181 | except tokenize.TokenError, ex: |
|
181 | except tokenize.TokenError, ex: | |
182 | msg = ex[0] |
|
182 | msg = ex[0] | |
183 | line = ex[1][0] |
|
183 | line = ex[1][0] | |
184 | self.out.write("%s\n\n*** ERROR: %s%s%s\n" % |
|
184 | self.out.write("%s\n\n*** ERROR: %s%s%s\n" % | |
185 | (colors[token.ERRORTOKEN], |
|
185 | (colors[token.ERRORTOKEN], | |
186 | msg, self.raw[self.lines[line]:], |
|
186 | msg, self.raw[self.lines[line]:], | |
187 | colors.normal) |
|
187 | colors.normal) | |
188 | ) |
|
188 | ) | |
189 | error = True |
|
189 | error = True | |
190 | self.out.write(colors.normal+'\n') |
|
190 | self.out.write(colors.normal+'\n') | |
191 | if string_output: |
|
191 | if string_output: | |
192 | output = self.out.getvalue() |
|
192 | output = self.out.getvalue() | |
193 | self.out = out_old |
|
193 | self.out = out_old | |
194 | return (output, error) |
|
194 | return (output, error) | |
195 | return (None, error) |
|
195 | return (None, error) | |
196 |
|
196 | |||
197 | def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line): |
|
197 | def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line): | |
198 | """ Token handler, with syntax highlighting.""" |
|
198 | """ Token handler, with syntax highlighting.""" | |
199 |
|
199 | |||
200 | # local shorthands |
|
200 | # local shorthands | |
201 | colors = self.colors |
|
201 | colors = self.colors | |
202 | owrite = self.out.write |
|
202 | owrite = self.out.write | |
203 |
|
203 | |||
204 | # line separator, so this works across platforms |
|
204 | # line separator, so this works across platforms | |
205 | linesep = os.linesep |
|
205 | linesep = os.linesep | |
206 |
|
206 | |||
207 | # calculate new positions |
|
207 | # calculate new positions | |
208 | oldpos = self.pos |
|
208 | oldpos = self.pos | |
209 | newpos = self.lines[srow] + scol |
|
209 | newpos = self.lines[srow] + scol | |
210 | self.pos = newpos + len(toktext) |
|
210 | self.pos = newpos + len(toktext) | |
211 |
|
211 | |||
212 | # handle newlines |
|
212 | # handle newlines | |
213 | if toktype in [token.NEWLINE, tokenize.NL]: |
|
213 | if toktype in [token.NEWLINE, tokenize.NL]: | |
214 | owrite(linesep) |
|
214 | owrite(linesep) | |
215 | return |
|
215 | return | |
216 |
|
216 | |||
217 | # send the original whitespace, if needed |
|
217 | # send the original whitespace, if needed | |
218 | if newpos > oldpos: |
|
218 | if newpos > oldpos: | |
219 | owrite(self.raw[oldpos:newpos]) |
|
219 | owrite(self.raw[oldpos:newpos]) | |
220 |
|
220 | |||
221 | # skip indenting tokens |
|
221 | # skip indenting tokens | |
222 | if toktype in [token.INDENT, token.DEDENT]: |
|
222 | if toktype in [token.INDENT, token.DEDENT]: | |
223 | self.pos = newpos |
|
223 | self.pos = newpos | |
224 | return |
|
224 | return | |
225 |
|
225 | |||
226 | # map token type to a color group |
|
226 | # map token type to a color group | |
227 | if token.LPAR <= toktype and toktype <= token.OP: |
|
227 | if token.LPAR <= toktype and toktype <= token.OP: | |
228 | toktype = token.OP |
|
228 | toktype = token.OP | |
229 | elif toktype == token.NAME and keyword.iskeyword(toktext): |
|
229 | elif toktype == token.NAME and keyword.iskeyword(toktext): | |
230 | toktype = _KEYWORD |
|
230 | toktype = _KEYWORD | |
231 | color = colors.get(toktype, colors[_TEXT]) |
|
231 | color = colors.get(toktype, colors[_TEXT]) | |
232 |
|
232 | |||
233 | #print '<%s>' % toktext, # dbg |
|
233 | #print '<%s>' % toktext, # dbg | |
234 |
|
234 | |||
235 | # Triple quoted strings must be handled carefully so that backtracking |
|
235 | # Triple quoted strings must be handled carefully so that backtracking | |
236 | # in pagers works correctly. We need color terminators on _each_ line. |
|
236 | # in pagers works correctly. We need color terminators on _each_ line. | |
237 | if linesep in toktext: |
|
237 | if linesep in toktext: | |
238 | toktext = toktext.replace(linesep, '%s%s%s' % |
|
238 | toktext = toktext.replace(linesep, '%s%s%s' % | |
239 | (colors.normal,linesep,color)) |
|
239 | (colors.normal,linesep,color)) | |
240 |
|
240 | |||
241 | # send text |
|
241 | # send text | |
242 | owrite('%s%s%s' % (color,toktext,colors.normal)) |
|
242 | owrite('%s%s%s' % (color,toktext,colors.normal)) | |
243 |
|
243 | |||
244 | def main(argv=None): |
|
244 | def main(argv=None): | |
245 | """Run as a command-line script: colorize a python file or stdin using ANSI |
|
245 | """Run as a command-line script: colorize a python file or stdin using ANSI | |
246 | color escapes and print to stdout. |
|
246 | color escapes and print to stdout. | |
247 |
|
247 | |||
248 | Inputs: |
|
248 | Inputs: | |
249 |
|
249 | |||
250 | - argv(None): a list of strings like sys.argv[1:] giving the command-line |
|
250 | - argv(None): a list of strings like sys.argv[1:] giving the command-line | |
251 | arguments. If None, use sys.argv[1:]. |
|
251 | arguments. If None, use sys.argv[1:]. | |
252 | """ |
|
252 | """ | |
253 |
|
253 | |||
254 | usage_msg = """%prog [options] [filename] |
|
254 | usage_msg = """%prog [options] [filename] | |
255 |
|
255 | |||
256 | Colorize a python file or stdin using ANSI color escapes and print to stdout. |
|
256 | Colorize a python file or stdin using ANSI color escapes and print to stdout. | |
257 | If no filename is given, or if filename is -, read standard input.""" |
|
257 | If no filename is given, or if filename is -, read standard input.""" | |
258 |
|
258 | |||
259 | parser = optparse.OptionParser(usage=usage_msg) |
|
259 | parser = optparse.OptionParser(usage=usage_msg) | |
260 | newopt = parser.add_option |
|
260 | newopt = parser.add_option | |
261 | newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store', |
|
261 | newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store', | |
262 | choices=['Linux','LightBG','NoColor'],default=_scheme_default, |
|
262 | choices=['Linux','LightBG','NoColor'],default=_scheme_default, | |
263 | help="give the color scheme to use. Currently only 'Linux'\ |
|
263 | help="give the color scheme to use. Currently only 'Linux'\ | |
264 | (default) and 'LightBG' and 'NoColor' are implemented (give without\ |
|
264 | (default) and 'LightBG' and 'NoColor' are implemented (give without\ | |
265 | quotes)") |
|
265 | quotes)") | |
266 |
|
266 | |||
267 | opts,args = parser.parse_args(argv) |
|
267 | opts,args = parser.parse_args(argv) | |
268 |
|
268 | |||
269 | if len(args) > 1: |
|
269 | if len(args) > 1: | |
270 | parser.error("you must give at most one filename.") |
|
270 | parser.error("you must give at most one filename.") | |
271 |
|
271 | |||
272 | if len(args) == 0: |
|
272 | if len(args) == 0: | |
273 | fname = '-' # no filename given; setup to read from stdin |
|
273 | fname = '-' # no filename given; setup to read from stdin | |
274 | else: |
|
274 | else: | |
275 | fname = args[0] |
|
275 | fname = args[0] | |
276 |
|
276 | |||
277 | if fname == '-': |
|
277 | if fname == '-': | |
278 | stream = sys.stdin |
|
278 | stream = sys.stdin | |
279 | else: |
|
279 | else: | |
280 | stream = file(fname) |
|
280 | try: | |
|
281 | stream = file(fname) | |||
|
282 | except IOError,msg: | |||
|
283 | print >> sys.stderr, msg | |||
|
284 | sys.exit(1) | |||
281 |
|
285 | |||
282 | parser = Parser() |
|
286 | parser = Parser() | |
283 |
|
287 | |||
284 | # we need nested try blocks because pre-2.5 python doesn't support unified |
|
288 | # we need nested try blocks because pre-2.5 python doesn't support unified | |
285 | # try-except-finally |
|
289 | # try-except-finally | |
286 | try: |
|
290 | try: | |
287 | try: |
|
291 | try: | |
288 | # write colorized version to stdout |
|
292 | # write colorized version to stdout | |
289 | parser.format(stream.read(),scheme=opts.scheme_name) |
|
293 | parser.format(stream.read(),scheme=opts.scheme_name) | |
290 | except IOError,msg: |
|
294 | except IOError,msg: | |
291 | # if user reads through a pager and quits, don't print traceback |
|
295 | # if user reads through a pager and quits, don't print traceback | |
292 | if msg.args != (32,'Broken pipe'): |
|
296 | if msg.args != (32,'Broken pipe'): | |
293 | raise |
|
297 | raise | |
294 | finally: |
|
298 | finally: | |
295 | if stream is not sys.stdin: |
|
299 | if stream is not sys.stdin: | |
296 | stream.close() # in case a non-handled exception happened above |
|
300 | stream.close() # in case a non-handled exception happened above | |
297 |
|
301 | |||
298 | if __name__ == "__main__": |
|
302 | if __name__ == "__main__": | |
299 | main() |
|
303 | main() |
General Comments 0
You need to be logged in to leave comments.
Login now