##// END OF EJS Templates
Paul Mueller: pycolorize considers 'no arguments' as 'read data from stdin'"
vivainio -
Show More
@@ -1,284 +1,301 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 Class and program to colorize python source code for ANSI terminals.
3 Class and program to colorize python source code for ANSI terminals.
4
4
5 Based on an HTML code highlighter by Jurgen Hermann found at:
5 Based on an HTML code highlighter by Jurgen Hermann found at:
6 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
6 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
7
7
8 Modifications by Fernando Perez (fperez@colorado.edu).
8 Modifications by Fernando Perez (fperez@colorado.edu).
9
9
10 Information on the original HTML highlighter follows:
10 Information on the original HTML highlighter follows:
11
11
12 MoinMoin - Python Source Parser
12 MoinMoin - Python Source Parser
13
13
14 Title:olorize Python source using the built-in tokenizer
14 Title:olorize Python source using the built-in tokenizer
15
15
16 Submitter: Jurgen Hermann
16 Submitter: Jurgen Hermann
17 Last Updated:2001/04/06
17 Last Updated:2001/04/06
18
18
19 Version no:1.2
19 Version no:1.2
20
20
21 Description:
21 Description:
22
22
23 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
23 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
24 Python source code to HTML markup, rendering comments, keywords,
24 Python source code to HTML markup, rendering comments, keywords,
25 operators, numeric and string literals in different colors.
25 operators, numeric and string literals in different colors.
26
26
27 It shows how to use the built-in keyword, token and tokenize modules to
27 It shows how to use the built-in keyword, token and tokenize modules to
28 scan Python source code and re-emit it with no changes to its original
28 scan Python source code and re-emit it with no changes to its original
29 formatting (which is the hard part).
29 formatting (which is the hard part).
30
30
31 $Id: PyColorize.py 2341 2007-05-15 14:44:30Z vivainio $"""
31 $Id: PyColorize.py 2342 2007-05-15 14:46:58Z vivainio $"""
32
32
33 __all__ = ['ANSICodeColors','Parser']
33 __all__ = ['ANSICodeColors','Parser']
34
34
35 _scheme_default = 'Linux'
35 _scheme_default = 'Linux'
36
36
37 # Imports
37 # Imports
38 import cStringIO
38 import cStringIO
39 import keyword
39 import keyword
40 import os
40 import os
41 import optparse
41 import optparse
42 import string
42 import string
43 import sys
43 import sys
44 import token
44 import token
45 import tokenize
45 import tokenize
46
46
47 from IPython.ColorANSI import *
47 from IPython.ColorANSI import *
48
48
49 #############################################################################
49 #############################################################################
50 ### Python Source Parser (does Hilighting)
50 ### Python Source Parser (does Hilighting)
51 #############################################################################
51 #############################################################################
52
52
53 _KEYWORD = token.NT_OFFSET + 1
53 _KEYWORD = token.NT_OFFSET + 1
54 _TEXT = token.NT_OFFSET + 2
54 _TEXT = token.NT_OFFSET + 2
55
55
56 #****************************************************************************
56 #****************************************************************************
57 # Builtin color schemes
57 # Builtin color schemes
58
58
59 Colors = TermColors # just a shorthand
59 Colors = TermColors # just a shorthand
60
60
61 # Build a few color schemes
61 # Build a few color schemes
62 NoColor = ColorScheme(
62 NoColor = ColorScheme(
63 'NoColor',{
63 'NoColor',{
64 token.NUMBER : Colors.NoColor,
64 token.NUMBER : Colors.NoColor,
65 token.OP : Colors.NoColor,
65 token.OP : Colors.NoColor,
66 token.STRING : Colors.NoColor,
66 token.STRING : Colors.NoColor,
67 tokenize.COMMENT : Colors.NoColor,
67 tokenize.COMMENT : Colors.NoColor,
68 token.NAME : Colors.NoColor,
68 token.NAME : Colors.NoColor,
69 token.ERRORTOKEN : Colors.NoColor,
69 token.ERRORTOKEN : Colors.NoColor,
70
70
71 _KEYWORD : Colors.NoColor,
71 _KEYWORD : Colors.NoColor,
72 _TEXT : Colors.NoColor,
72 _TEXT : Colors.NoColor,
73
73
74 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
74 'normal' : Colors.NoColor # color off (usu. Colors.Normal)
75 } )
75 } )
76
76
77 LinuxColors = ColorScheme(
77 LinuxColors = ColorScheme(
78 'Linux',{
78 'Linux',{
79 token.NUMBER : Colors.LightCyan,
79 token.NUMBER : Colors.LightCyan,
80 token.OP : Colors.Yellow,
80 token.OP : Colors.Yellow,
81 token.STRING : Colors.LightBlue,
81 token.STRING : Colors.LightBlue,
82 tokenize.COMMENT : Colors.LightRed,
82 tokenize.COMMENT : Colors.LightRed,
83 token.NAME : Colors.White,
83 token.NAME : Colors.White,
84 token.ERRORTOKEN : Colors.Red,
84 token.ERRORTOKEN : Colors.Red,
85
85
86 _KEYWORD : Colors.LightGreen,
86 _KEYWORD : Colors.LightGreen,
87 _TEXT : Colors.Yellow,
87 _TEXT : Colors.Yellow,
88
88
89 'normal' : Colors.Normal # color off (usu. Colors.Normal)
89 'normal' : Colors.Normal # color off (usu. Colors.Normal)
90 } )
90 } )
91
91
92 LightBGColors = ColorScheme(
92 LightBGColors = ColorScheme(
93 'LightBG',{
93 'LightBG',{
94 token.NUMBER : Colors.Cyan,
94 token.NUMBER : Colors.Cyan,
95 token.OP : Colors.Blue,
95 token.OP : Colors.Blue,
96 token.STRING : Colors.Blue,
96 token.STRING : Colors.Blue,
97 tokenize.COMMENT : Colors.Red,
97 tokenize.COMMENT : Colors.Red,
98 token.NAME : Colors.Black,
98 token.NAME : Colors.Black,
99 token.ERRORTOKEN : Colors.Red,
99 token.ERRORTOKEN : Colors.Red,
100
100
101 _KEYWORD : Colors.Green,
101 _KEYWORD : Colors.Green,
102 _TEXT : Colors.Blue,
102 _TEXT : Colors.Blue,
103
103
104 'normal' : Colors.Normal # color off (usu. Colors.Normal)
104 'normal' : Colors.Normal # color off (usu. Colors.Normal)
105 } )
105 } )
106
106
107 # Build table of color schemes (needed by the parser)
107 # Build table of color schemes (needed by the parser)
108 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
108 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
109 _scheme_default)
109 _scheme_default)
110
110
111 class Parser:
111 class Parser:
112 """ Format colored Python source.
112 """ Format colored Python source.
113 """
113 """
114
114
115 def __init__(self, color_table=None,out = sys.stdout):
115 def __init__(self, color_table=None,out = sys.stdout):
116 """ Create a parser with a specified color table and output channel.
116 """ Create a parser with a specified color table and output channel.
117
117
118 Call format() to process code.
118 Call format() to process code.
119 """
119 """
120 self.color_table = color_table and color_table or ANSICodeColors
120 self.color_table = color_table and color_table or ANSICodeColors
121 self.out = out
121 self.out = out
122
122
123 def format(self, raw, out = None, scheme = ''):
123 def format(self, raw, out = None, scheme = ''):
124 return self.format2(raw, out, scheme)[0]
124 return self.format2(raw, out, scheme)[0]
125
125
126 def format2(self, raw, out = None, scheme = ''):
126 def format2(self, raw, out = None, scheme = ''):
127 """ Parse and send the colored source.
127 """ Parse and send the colored source.
128
128
129 If out and scheme are not specified, the defaults (given to
129 If out and scheme are not specified, the defaults (given to
130 constructor) are used.
130 constructor) are used.
131
131
132 out should be a file-type object. Optionally, out can be given as the
132 out should be a file-type object. Optionally, out can be given as the
133 string 'str' and the parser will automatically return the output in a
133 string 'str' and the parser will automatically return the output in a
134 string."""
134 string."""
135
135
136 string_output = 0
136 string_output = 0
137 if out == 'str' or self.out == 'str' or \
137 if out == 'str' or self.out == 'str' or \
138 isinstance(self.out,cStringIO.OutputType):
138 isinstance(self.out,cStringIO.OutputType):
139 # XXX - I don't really like this state handling logic, but at this
139 # XXX - I don't really like this state handling logic, but at this
140 # point I don't want to make major changes, so adding the
140 # point I don't want to make major changes, so adding the
141 # isinstance() check is the simplest I can do to ensure correct
141 # isinstance() check is the simplest I can do to ensure correct
142 # behavior.
142 # behavior.
143 out_old = self.out
143 out_old = self.out
144 self.out = cStringIO.StringIO()
144 self.out = cStringIO.StringIO()
145 string_output = 1
145 string_output = 1
146 elif out is not None:
146 elif out is not None:
147 self.out = out
147 self.out = out
148
148
149 # Fast return of the unmodified input for NoColor scheme
149 # Fast return of the unmodified input for NoColor scheme
150 if scheme == 'NoColor':
150 if scheme == 'NoColor':
151 error = False
151 error = False
152 self.out.write(raw)
152 self.out.write(raw)
153 if string_output:
153 if string_output:
154 return raw,error
154 return raw,error
155 else:
155 else:
156 return None,error
156 return None,error
157
157
158 # local shorthands
158 # local shorthands
159 colors = self.color_table[scheme].colors
159 colors = self.color_table[scheme].colors
160 self.colors = colors # put in object so __call__ sees it
160 self.colors = colors # put in object so __call__ sees it
161
161
162 # Remove trailing whitespace and normalize tabs
162 # Remove trailing whitespace and normalize tabs
163 self.raw = raw.expandtabs().rstrip()
163 self.raw = raw.expandtabs().rstrip()
164
164
165 # store line offsets in self.lines
165 # store line offsets in self.lines
166 self.lines = [0, 0]
166 self.lines = [0, 0]
167 pos = 0
167 pos = 0
168 raw_find = self.raw.find
168 raw_find = self.raw.find
169 lines_append = self.lines.append
169 lines_append = self.lines.append
170 while 1:
170 while 1:
171 pos = raw_find('\n', pos) + 1
171 pos = raw_find('\n', pos) + 1
172 if not pos: break
172 if not pos: break
173 lines_append(pos)
173 lines_append(pos)
174 lines_append(len(self.raw))
174 lines_append(len(self.raw))
175
175
176 # parse the source and write it
176 # parse the source and write it
177 self.pos = 0
177 self.pos = 0
178 text = cStringIO.StringIO(self.raw)
178 text = cStringIO.StringIO(self.raw)
179
179
180 error = False
180 error = False
181 try:
181 try:
182 tokenize.tokenize(text.readline, self)
182 tokenize.tokenize(text.readline, self)
183 except tokenize.TokenError, ex:
183 except tokenize.TokenError, ex:
184 msg = ex[0]
184 msg = ex[0]
185 line = ex[1][0]
185 line = ex[1][0]
186 self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
186 self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
187 (colors[token.ERRORTOKEN],
187 (colors[token.ERRORTOKEN],
188 msg, self.raw[self.lines[line]:],
188 msg, self.raw[self.lines[line]:],
189 colors.normal)
189 colors.normal)
190 )
190 )
191 error = True
191 error = True
192 self.out.write(colors.normal+'\n')
192 self.out.write(colors.normal+'\n')
193 if string_output:
193 if string_output:
194 output = self.out.getvalue()
194 output = self.out.getvalue()
195 self.out = out_old
195 self.out = out_old
196 return (output, error)
196 return (output, error)
197 return (None, error)
197 return (None, error)
198
198
199 def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
199 def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
200 """ Token handler, with syntax highlighting."""
200 """ Token handler, with syntax highlighting."""
201
201
202 # local shorthands
202 # local shorthands
203 colors = self.colors
203 colors = self.colors
204 owrite = self.out.write
204 owrite = self.out.write
205
205
206 # line separator, so this works across platforms
206 # line separator, so this works across platforms
207 linesep = os.linesep
207 linesep = os.linesep
208
208
209 # calculate new positions
209 # calculate new positions
210 oldpos = self.pos
210 oldpos = self.pos
211 newpos = self.lines[srow] + scol
211 newpos = self.lines[srow] + scol
212 self.pos = newpos + len(toktext)
212 self.pos = newpos + len(toktext)
213
213
214 # handle newlines
214 # handle newlines
215 if toktype in [token.NEWLINE, tokenize.NL]:
215 if toktype in [token.NEWLINE, tokenize.NL]:
216 owrite(linesep)
216 owrite(linesep)
217 return
217 return
218
218
219 # send the original whitespace, if needed
219 # send the original whitespace, if needed
220 if newpos > oldpos:
220 if newpos > oldpos:
221 owrite(self.raw[oldpos:newpos])
221 owrite(self.raw[oldpos:newpos])
222
222
223 # skip indenting tokens
223 # skip indenting tokens
224 if toktype in [token.INDENT, token.DEDENT]:
224 if toktype in [token.INDENT, token.DEDENT]:
225 self.pos = newpos
225 self.pos = newpos
226 return
226 return
227
227
228 # map token type to a color group
228 # map token type to a color group
229 if token.LPAR <= toktype and toktype <= token.OP:
229 if token.LPAR <= toktype and toktype <= token.OP:
230 toktype = token.OP
230 toktype = token.OP
231 elif toktype == token.NAME and keyword.iskeyword(toktext):
231 elif toktype == token.NAME and keyword.iskeyword(toktext):
232 toktype = _KEYWORD
232 toktype = _KEYWORD
233 color = colors.get(toktype, colors[_TEXT])
233 color = colors.get(toktype, colors[_TEXT])
234
234
235 #print '<%s>' % toktext, # dbg
235 #print '<%s>' % toktext, # dbg
236
236
237 # Triple quoted strings must be handled carefully so that backtracking
237 # Triple quoted strings must be handled carefully so that backtracking
238 # in pagers works correctly. We need color terminators on _each_ line.
238 # in pagers works correctly. We need color terminators on _each_ line.
239 if linesep in toktext:
239 if linesep in toktext:
240 toktext = toktext.replace(linesep, '%s%s%s' %
240 toktext = toktext.replace(linesep, '%s%s%s' %
241 (colors.normal,linesep,color))
241 (colors.normal,linesep,color))
242
242
243 # send text
243 # send text
244 owrite('%s%s%s' % (color,toktext,colors.normal))
244 owrite('%s%s%s' % (color,toktext,colors.normal))
245
245
246 def main(argv=None):
246 def main(argv=None):
247 """Run as a command-line script: colorize a python file using ANSI color
247 """Run as a command-line script: colorize a python file or stdin using ANSI
248 escapes and print to stdout.
248 color escapes and print to stdout.
249
249
250 Inputs:
250 Inputs:
251
251
252 - argv(None): a list of strings like sys.argv[1:] giving the command-line
252 - argv(None): a list of strings like sys.argv[1:] giving the command-line
253 arguments. If None, use sys.argv[1:].
253 arguments. If None, use sys.argv[1:].
254 """
254 """
255
255
256 usage_msg = """%prog [options] filename
256 usage_msg = """%prog [options] [filename]
257
257
258 Colorize a python file using ANSI color escapes and print to stdout."""
258 Colorize a python file or stdin using ANSI color escapes and print to stdout.
259 If no filename is given, or if filename is -, read standard input."""
259
260
260 parser = optparse.OptionParser(usage=usage_msg)
261 parser = optparse.OptionParser(usage=usage_msg)
261 newopt = parser.add_option
262 newopt = parser.add_option
262 newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
263 newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
263 choices=['Linux','LightBG','NoColor'],default=_scheme_default,
264 choices=['Linux','LightBG','NoColor'],default=_scheme_default,
264 help="give the color scheme to use. Currently only 'Linux'\
265 help="give the color scheme to use. Currently only 'Linux'\
265 (default) and 'LightBG' and 'NoColor' are implemented (give without\
266 (default) and 'LightBG' and 'NoColor' are implemented (give without\
266 quotes)")
267 quotes)")
267
268
268 opts,args = parser.parse_args(argv)
269 opts,args = parser.parse_args(argv)
269
270
270 if len(args) != 1:
271 if len(args) > 1:
271 parser.error("you must give one filename.")
272 parser.error("you must give at most one filename.")
273
274 if len(args) == 0:
275 fname = '-' # no filename given; setup to read from stdin
276 else:
277 fname = args[0]
278
279 if fname == '-':
280 stream = sys.stdin
281 else:
282 stream = file(fname)
272
283
273 # write colorized version to stdout
274 fname = args[0]
275 parser = Parser()
284 parser = Parser()
285
286 # we need nested try blocks because pre-2.5 python doesn't support unified
287 # try-except-finally
276 try:
288 try:
277 parser.format(file(fname).read(),scheme=opts.scheme_name)
289 try:
278 except IOError,msg:
290 # write colorized version to stdout
279 # if user reads through a pager and quits, don't print traceback
291 parser.format(stream.read(),scheme=opts.scheme_name)
280 if msg.args != (32,'Broken pipe'):
292 except IOError,msg:
281 raise
293 # if user reads through a pager and quits, don't print traceback
294 if msg.args != (32,'Broken pipe'):
295 raise
296 finally:
297 if stream is not sys.stdin:
298 stream.close() # in case a non-handled exception happened above
282
299
283 if __name__ == "__main__":
300 if __name__ == "__main__":
284 main()
301 main()
@@ -1,36 +1,38 b''
1 .\" Hey, EMACS: -*- nroff -*-
1 .\" Hey, EMACS: -*- nroff -*-
2 .\" First parameter, NAME, should be all caps
2 .\" First parameter, NAME, should be all caps
3 .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
3 .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
4 .\" other parameters are allowed: see man(7), man(1)
4 .\" other parameters are allowed: see man(7), man(1)
5 .TH PYCOLOR 1 "May 12, 2007"
5 .TH PYCOLOR 1 "May 12, 2007"
6 .\" Please adjust this date whenever revising the manpage.
6 .\" Please adjust this date whenever revising the manpage.
7 .\"
7 .\"
8 .\" Some roff macros, for reference:
8 .\" Some roff macros, for reference:
9 .\" .nh disable hyphenation
9 .\" .nh disable hyphenation
10 .\" .hy enable hyphenation
10 .\" .hy enable hyphenation
11 .\" .ad l left justify
11 .\" .ad l left justify
12 .\" .ad b justify to both left and right margins
12 .\" .ad b justify to both left and right margins
13 .\" .nf disable filling
13 .\" .nf disable filling
14 .\" .fi enable filling
14 .\" .fi enable filling
15 .\" .br insert line break
15 .\" .br insert line break
16 .\" .sp <n> insert n+1 empty lines
16 .\" .sp <n> insert n+1 empty lines
17 .\" for manpage-specific macros, see man(7)
17 .\" for manpage-specific macros, see man(7)
18 .SH NAME
18 .SH NAME
19 pycolor \- Colorize a python file using ANSI and print to stdout.
19 pycolor \- Colorize a python file or stdin using ANSI and print to stdout.
20 .SH SYNOPSIS
20 .SH SYNOPSIS
21 .B pycolor
21 .B pycolor
22 .RI [ options ] " file"
22 .RI [ options ]
23 .RI [ file ]
23 .SH DESCRIPTION
24 .SH DESCRIPTION
24 Prints a colorized version of the input file to standard out.
25 Prints a colorized version of the input file (or standard input if no file is
26 given, or the file name - is given) to standard out.
25 .SH OPTIONS
27 .SH OPTIONS
26 .TP
28 .TP
27 .B \-h, \-\-help
29 .B \-h, \-\-help
28 Output a brief help message.
30 Output a brief help message.
29 .TP
31 .TP
30 .B \-s, \-\-scheme <scheme>
32 .B \-s, \-\-scheme <scheme>
31 Give the color scheme to use. Currently only Linux (default),
33 Give the color scheme to use. Currently only Linux (default),
32 LightBG, and NOColor are implemented.
34 LightBG, and NOColor are implemented.
33 .SH AUTHOR
35 .SH AUTHOR
34 pycolor was written by Fernando Perez <fperez@colorado.edu>.
36 pycolor was written by Fernando Perez <fperez@colorado.edu>.
35 This manual page was written by Jack Moffitt <jack@xiph.org>,
37 This manual page was written by Jack Moffitt <jack@xiph.org>,
36 for the Debian project (but may be used by others).
38 for the Debian project (but may be used by others).
General Comments 0
You need to be logged in to leave comments. Login now