##// END OF EJS Templates
use Colors.Normal for NAMEs in PyColorize...
MinRK -
Show More
@@ -1,303 +1,303 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 Class and program to colorize python source code for ANSI terminals.
3 Class and program to colorize python source code for ANSI terminals.
4
4
5 Based on an HTML code highlighter by Jurgen Hermann found at:
5 Based on an HTML code highlighter by Jurgen Hermann found at:
6 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
6 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
7
7
8 Modifications by Fernando Perez (fperez@colorado.edu).
8 Modifications by Fernando Perez (fperez@colorado.edu).
9
9
10 Information on the original HTML highlighter follows:
10 Information on the original HTML highlighter follows:
11
11
12 MoinMoin - Python Source Parser
12 MoinMoin - Python Source Parser
13
13
14 Title: Colorize Python source using the built-in tokenizer
14 Title: Colorize Python source using the built-in tokenizer
15
15
16 Submitter: Jurgen Hermann
16 Submitter: Jurgen Hermann
17 Last Updated:2001/04/06
17 Last Updated:2001/04/06
18
18
19 Version no:1.2
19 Version no:1.2
20
20
21 Description:
21 Description:
22
22
23 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
23 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
24 Python source code to HTML markup, rendering comments, keywords,
24 Python source code to HTML markup, rendering comments, keywords,
25 operators, numeric and string literals in different colors.
25 operators, numeric and string literals in different colors.
26
26
27 It shows how to use the built-in keyword, token and tokenize modules to
27 It shows how to use the built-in keyword, token and tokenize modules to
28 scan Python source code and re-emit it with no changes to its original
28 scan Python source code and re-emit it with no changes to its original
29 formatting (which is the hard part).
29 formatting (which is the hard part).
30 """
30 """
31
31
32 __all__ = ['ANSICodeColors','Parser']
32 __all__ = ['ANSICodeColors','Parser']
33
33
34 _scheme_default = 'Linux'
34 _scheme_default = 'Linux'
35
35
36 # Imports
36 # Imports
37 import cStringIO
37 import cStringIO
38 import keyword
38 import keyword
39 import os
39 import os
40 import optparse
40 import optparse
41 import sys
41 import sys
42 import token
42 import token
43 import tokenize
43 import tokenize
44
44
45 from IPython.utils.coloransi import *
45 from IPython.utils.coloransi import *
46
46
47 #############################################################################
47 #############################################################################
48 ### Python Source Parser (does Hilighting)
48 ### Python Source Parser (does Hilighting)
49 #############################################################################
49 #############################################################################
50
50
51 _KEYWORD = token.NT_OFFSET + 1
51 _KEYWORD = token.NT_OFFSET + 1
52 _TEXT = token.NT_OFFSET + 2
52 _TEXT = token.NT_OFFSET + 2
53
53
54 #****************************************************************************
54 #****************************************************************************
55 # Builtin color schemes
55 # Builtin color schemes
56
56
57 Colors = TermColors # just a shorthand
57 Colors = TermColors # just a shorthand
58
58
59 # Build a few color schemes
59 # Build a few color schemes
60 NoColor = ColorScheme(
60 NoColor = ColorScheme(
61 'NoColor',{
61 'NoColor',{
62 token.NUMBER : Colors.NoColor,
62 token.NUMBER : Colors.NoColor,
63 token.OP : Colors.NoColor,
63 token.OP : Colors.NoColor,
64 token.STRING : Colors.NoColor,
64 token.STRING : Colors.NoColor,
65 tokenize.COMMENT : Colors.NoColor,
65 tokenize.COMMENT : Colors.NoColor,
66 token.NAME : Colors.NoColor,
66 token.NAME : Colors.NoColor,
67 token.ERRORTOKEN : Colors.NoColor,
67 token.ERRORTOKEN : Colors.NoColor,
68
68
69 _KEYWORD : Colors.NoColor,
69 _KEYWORD : Colors.NoColor,
70 _TEXT : Colors.NoColor,
70 _TEXT : Colors.NoColor,
71
71
72 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
72 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
73 } )
73 } )
74
74
75 LinuxColors = ColorScheme(
75 LinuxColors = ColorScheme(
76 'Linux',{
76 'Linux',{
77 token.NUMBER : Colors.LightCyan,
77 token.NUMBER : Colors.LightCyan,
78 token.OP : Colors.Yellow,
78 token.OP : Colors.Yellow,
79 token.STRING : Colors.LightBlue,
79 token.STRING : Colors.LightBlue,
80 tokenize.COMMENT : Colors.LightRed,
80 tokenize.COMMENT : Colors.LightRed,
81 token.NAME : Colors.White,
81 token.NAME : Colors.Normal,
82 token.ERRORTOKEN : Colors.Red,
82 token.ERRORTOKEN : Colors.Red,
83
83
84 _KEYWORD : Colors.LightGreen,
84 _KEYWORD : Colors.LightGreen,
85 _TEXT : Colors.Yellow,
85 _TEXT : Colors.Yellow,
86
86
87 'normal' : Colors.Normal # color off (usu. Colors.Normal)
87 'normal' : Colors.Normal # color off (usu. Colors.Normal)
88 } )
88 } )
89
89
90 LightBGColors = ColorScheme(
90 LightBGColors = ColorScheme(
91 'LightBG',{
91 'LightBG',{
92 token.NUMBER : Colors.Cyan,
92 token.NUMBER : Colors.Cyan,
93 token.OP : Colors.Blue,
93 token.OP : Colors.Blue,
94 token.STRING : Colors.Blue,
94 token.STRING : Colors.Blue,
95 tokenize.COMMENT : Colors.Red,
95 tokenize.COMMENT : Colors.Red,
96 token.NAME : Colors.Black,
96 token.NAME : Colors.Normal,
97 token.ERRORTOKEN : Colors.Red,
97 token.ERRORTOKEN : Colors.Red,
98
98
99 _KEYWORD : Colors.Green,
99 _KEYWORD : Colors.Green,
100 _TEXT : Colors.Blue,
100 _TEXT : Colors.Blue,
101
101
102 'normal' : Colors.Normal # color off (usu. Colors.Normal)
102 'normal' : Colors.Normal # color off (usu. Colors.Normal)
103 } )
103 } )
104
104
105 # Build table of color schemes (needed by the parser)
105 # Build table of color schemes (needed by the parser)
106 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
106 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
107 _scheme_default)
107 _scheme_default)
108
108
109 class Parser:
109 class Parser:
110 """ Format colored Python source.
110 """ Format colored Python source.
111 """
111 """
112
112
113 def __init__(self, color_table=None,out = sys.stdout):
113 def __init__(self, color_table=None,out = sys.stdout):
114 """ Create a parser with a specified color table and output channel.
114 """ Create a parser with a specified color table and output channel.
115
115
116 Call format() to process code.
116 Call format() to process code.
117 """
117 """
118 self.color_table = color_table and color_table or ANSICodeColors
118 self.color_table = color_table and color_table or ANSICodeColors
119 self.out = out
119 self.out = out
120
120
121 def format(self, raw, out = None, scheme = ''):
121 def format(self, raw, out = None, scheme = ''):
122 return self.format2(raw, out, scheme)[0]
122 return self.format2(raw, out, scheme)[0]
123
123
124 def format2(self, raw, out = None, scheme = ''):
124 def format2(self, raw, out = None, scheme = ''):
125 """ Parse and send the colored source.
125 """ Parse and send the colored source.
126
126
127 If out and scheme are not specified, the defaults (given to
127 If out and scheme are not specified, the defaults (given to
128 constructor) are used.
128 constructor) are used.
129
129
130 out should be a file-type object. Optionally, out can be given as the
130 out should be a file-type object. Optionally, out can be given as the
131 string 'str' and the parser will automatically return the output in a
131 string 'str' and the parser will automatically return the output in a
132 string."""
132 string."""
133
133
134 string_output = 0
134 string_output = 0
135 if out == 'str' or self.out == 'str' or \
135 if out == 'str' or self.out == 'str' or \
136 isinstance(self.out,cStringIO.OutputType):
136 isinstance(self.out,cStringIO.OutputType):
137 # XXX - I don't really like this state handling logic, but at this
137 # XXX - I don't really like this state handling logic, but at this
138 # point I don't want to make major changes, so adding the
138 # point I don't want to make major changes, so adding the
139 # isinstance() check is the simplest I can do to ensure correct
139 # isinstance() check is the simplest I can do to ensure correct
140 # behavior.
140 # behavior.
141 out_old = self.out
141 out_old = self.out
142 self.out = cStringIO.StringIO()
142 self.out = cStringIO.StringIO()
143 string_output = 1
143 string_output = 1
144 elif out is not None:
144 elif out is not None:
145 self.out = out
145 self.out = out
146
146
147 # Fast return of the unmodified input for NoColor scheme
147 # Fast return of the unmodified input for NoColor scheme
148 if scheme == 'NoColor':
148 if scheme == 'NoColor':
149 error = False
149 error = False
150 self.out.write(raw)
150 self.out.write(raw)
151 if string_output:
151 if string_output:
152 return raw,error
152 return raw,error
153 else:
153 else:
154 return None,error
154 return None,error
155
155
156 # local shorthands
156 # local shorthands
157 colors = self.color_table[scheme].colors
157 colors = self.color_table[scheme].colors
158 self.colors = colors # put in object so __call__ sees it
158 self.colors = colors # put in object so __call__ sees it
159
159
160 # Remove trailing whitespace and normalize tabs
160 # Remove trailing whitespace and normalize tabs
161 self.raw = raw.expandtabs().rstrip()
161 self.raw = raw.expandtabs().rstrip()
162
162
163 # store line offsets in self.lines
163 # store line offsets in self.lines
164 self.lines = [0, 0]
164 self.lines = [0, 0]
165 pos = 0
165 pos = 0
166 raw_find = self.raw.find
166 raw_find = self.raw.find
167 lines_append = self.lines.append
167 lines_append = self.lines.append
168 while 1:
168 while 1:
169 pos = raw_find('\n', pos) + 1
169 pos = raw_find('\n', pos) + 1
170 if not pos: break
170 if not pos: break
171 lines_append(pos)
171 lines_append(pos)
172 lines_append(len(self.raw))
172 lines_append(len(self.raw))
173
173
174 # parse the source and write it
174 # parse the source and write it
175 self.pos = 0
175 self.pos = 0
176 text = cStringIO.StringIO(self.raw)
176 text = cStringIO.StringIO(self.raw)
177
177
178 error = False
178 error = False
179 try:
179 try:
180 tokenize.tokenize(text.readline, self)
180 tokenize.tokenize(text.readline, self)
181 except tokenize.TokenError, ex:
181 except tokenize.TokenError, ex:
182 msg = ex[0]
182 msg = ex[0]
183 line = ex[1][0]
183 line = ex[1][0]
184 self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
184 self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
185 (colors[token.ERRORTOKEN],
185 (colors[token.ERRORTOKEN],
186 msg, self.raw[self.lines[line]:],
186 msg, self.raw[self.lines[line]:],
187 colors.normal)
187 colors.normal)
188 )
188 )
189 error = True
189 error = True
190 self.out.write(colors.normal+'\n')
190 self.out.write(colors.normal+'\n')
191 if string_output:
191 if string_output:
192 output = self.out.getvalue()
192 output = self.out.getvalue()
193 self.out = out_old
193 self.out = out_old
194 return (output, error)
194 return (output, error)
195 return (None, error)
195 return (None, error)
196
196
197 def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
197 def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
198 """ Token handler, with syntax highlighting."""
198 """ Token handler, with syntax highlighting."""
199
199
200 # local shorthands
200 # local shorthands
201 colors = self.colors
201 colors = self.colors
202 owrite = self.out.write
202 owrite = self.out.write
203
203
204 # line separator, so this works across platforms
204 # line separator, so this works across platforms
205 linesep = os.linesep
205 linesep = os.linesep
206
206
207 # calculate new positions
207 # calculate new positions
208 oldpos = self.pos
208 oldpos = self.pos
209 newpos = self.lines[srow] + scol
209 newpos = self.lines[srow] + scol
210 self.pos = newpos + len(toktext)
210 self.pos = newpos + len(toktext)
211
211
212 # handle newlines
212 # handle newlines
213 if toktype in [token.NEWLINE, tokenize.NL]:
213 if toktype in [token.NEWLINE, tokenize.NL]:
214 owrite(linesep)
214 owrite(linesep)
215 return
215 return
216
216
217 # send the original whitespace, if needed
217 # send the original whitespace, if needed
218 if newpos > oldpos:
218 if newpos > oldpos:
219 owrite(self.raw[oldpos:newpos])
219 owrite(self.raw[oldpos:newpos])
220
220
221 # skip indenting tokens
221 # skip indenting tokens
222 if toktype in [token.INDENT, token.DEDENT]:
222 if toktype in [token.INDENT, token.DEDENT]:
223 self.pos = newpos
223 self.pos = newpos
224 return
224 return
225
225
226 # map token type to a color group
226 # map token type to a color group
227 if token.LPAR <= toktype and toktype <= token.OP:
227 if token.LPAR <= toktype and toktype <= token.OP:
228 toktype = token.OP
228 toktype = token.OP
229 elif toktype == token.NAME and keyword.iskeyword(toktext):
229 elif toktype == token.NAME and keyword.iskeyword(toktext):
230 toktype = _KEYWORD
230 toktype = _KEYWORD
231 color = colors.get(toktype, colors[_TEXT])
231 color = colors.get(toktype, colors[_TEXT])
232
232
233 #print '<%s>' % toktext, # dbg
233 #print '<%s>' % toktext, # dbg
234
234
235 # Triple quoted strings must be handled carefully so that backtracking
235 # Triple quoted strings must be handled carefully so that backtracking
236 # in pagers works correctly. We need color terminators on _each_ line.
236 # in pagers works correctly. We need color terminators on _each_ line.
237 if linesep in toktext:
237 if linesep in toktext:
238 toktext = toktext.replace(linesep, '%s%s%s' %
238 toktext = toktext.replace(linesep, '%s%s%s' %
239 (colors.normal,linesep,color))
239 (colors.normal,linesep,color))
240
240
241 # send text
241 # send text
242 owrite('%s%s%s' % (color,toktext,colors.normal))
242 owrite('%s%s%s' % (color,toktext,colors.normal))
243
243
244 def main(argv=None):
244 def main(argv=None):
245 """Run as a command-line script: colorize a python file or stdin using ANSI
245 """Run as a command-line script: colorize a python file or stdin using ANSI
246 color escapes and print to stdout.
246 color escapes and print to stdout.
247
247
248 Inputs:
248 Inputs:
249
249
250 - argv(None): a list of strings like sys.argv[1:] giving the command-line
250 - argv(None): a list of strings like sys.argv[1:] giving the command-line
251 arguments. If None, use sys.argv[1:].
251 arguments. If None, use sys.argv[1:].
252 """
252 """
253
253
254 usage_msg = """%prog [options] [filename]
254 usage_msg = """%prog [options] [filename]
255
255
256 Colorize a python file or stdin using ANSI color escapes and print to stdout.
256 Colorize a python file or stdin using ANSI color escapes and print to stdout.
257 If no filename is given, or if filename is -, read standard input."""
257 If no filename is given, or if filename is -, read standard input."""
258
258
259 parser = optparse.OptionParser(usage=usage_msg)
259 parser = optparse.OptionParser(usage=usage_msg)
260 newopt = parser.add_option
260 newopt = parser.add_option
261 newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
261 newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
262 choices=['Linux','LightBG','NoColor'],default=_scheme_default,
262 choices=['Linux','LightBG','NoColor'],default=_scheme_default,
263 help="give the color scheme to use. Currently only 'Linux'\
263 help="give the color scheme to use. Currently only 'Linux'\
264 (default) and 'LightBG' and 'NoColor' are implemented (give without\
264 (default) and 'LightBG' and 'NoColor' are implemented (give without\
265 quotes)")
265 quotes)")
266
266
267 opts,args = parser.parse_args(argv)
267 opts,args = parser.parse_args(argv)
268
268
269 if len(args) > 1:
269 if len(args) > 1:
270 parser.error("you must give at most one filename.")
270 parser.error("you must give at most one filename.")
271
271
272 if len(args) == 0:
272 if len(args) == 0:
273 fname = '-' # no filename given; setup to read from stdin
273 fname = '-' # no filename given; setup to read from stdin
274 else:
274 else:
275 fname = args[0]
275 fname = args[0]
276
276
277 if fname == '-':
277 if fname == '-':
278 stream = sys.stdin
278 stream = sys.stdin
279 else:
279 else:
280 try:
280 try:
281 stream = file(fname)
281 stream = file(fname)
282 except IOError,msg:
282 except IOError,msg:
283 print >> sys.stderr, msg
283 print >> sys.stderr, msg
284 sys.exit(1)
284 sys.exit(1)
285
285
286 parser = Parser()
286 parser = Parser()
287
287
288 # we need nested try blocks because pre-2.5 python doesn't support unified
288 # we need nested try blocks because pre-2.5 python doesn't support unified
289 # try-except-finally
289 # try-except-finally
290 try:
290 try:
291 try:
291 try:
292 # write colorized version to stdout
292 # write colorized version to stdout
293 parser.format(stream.read(),scheme=opts.scheme_name)
293 parser.format(stream.read(),scheme=opts.scheme_name)
294 except IOError,msg:
294 except IOError,msg:
295 # if user reads through a pager and quits, don't print traceback
295 # if user reads through a pager and quits, don't print traceback
296 if msg.args != (32,'Broken pipe'):
296 if msg.args != (32,'Broken pipe'):
297 raise
297 raise
298 finally:
298 finally:
299 if stream is not sys.stdin:
299 if stream is not sys.stdin:
300 stream.close() # in case a non-handled exception happened above
300 stream.close() # in case a non-handled exception happened above
301
301
302 if __name__ == "__main__":
302 if __name__ == "__main__":
303 main()
303 main()
General Comments 0
You need to be logged in to leave comments. Login now