PyColorize.py
374 lines
| 11.6 KiB
| text/x-python
|
PythonLexer
|
r0 | # -*- coding: utf-8 -*- | ||
""" | ||||
|
r1853 | Class and program to colorize python source code for ANSI terminals. | ||
|
r0 | |||
|
r1853 | Based on an HTML code highlighter by Jurgen Hermann found at: | ||
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298 | ||||
|
r0 | |||
|
r1853 | Modifications by Fernando Perez (fperez@colorado.edu). | ||
|
r0 | |||
|
r1853 | Information on the original HTML highlighter follows: | ||
|
r0 | |||
|
r1853 | MoinMoin - Python Source Parser | ||
|
r0 | |||
|
r1853 | Title: Colorize Python source using the built-in tokenizer | ||
|
r0 | |||
|
r1853 | Submitter: Jurgen Hermann | ||
Last Updated:2001/04/06 | ||||
|
r0 | |||
|
r1853 | Version no:1.2 | ||
|
r0 | |||
|
r1853 | Description: | ||
This code is part of MoinMoin (http://moin.sourceforge.net/) and converts | ||||
Python source code to HTML markup, rendering comments, keywords, | ||||
operators, numeric and string literals in different colors. | ||||
It shows how to use the built-in keyword, token and tokenize modules to | ||||
scan Python source code and re-emit it with no changes to its original | ||||
formatting (which is the hard part). | ||||
""" | ||||
|
r7817 | from __future__ import print_function | ||
|
r13365 | from __future__ import absolute_import | ||
|
r8299 | from __future__ import unicode_literals | ||
|
r0 | __all__ = ['ANSICodeColors','Parser'] | ||
_scheme_default = 'Linux' | ||||
|
r7817 | |||
|
r0 | # Imports | ||
|
r52 | import keyword | ||
import os | ||||
import sys | ||||
import token | ||||
import tokenize | ||||
|
r0 | |||
|
r4758 | try: | ||
generate_tokens = tokenize.generate_tokens | ||||
except AttributeError: | ||||
# Python 3. Note that we use the undocumented _tokenize because it expects | ||||
# strings, not bytes. See also Python issue #9969. | ||||
generate_tokens = tokenize._tokenize | ||||
|
r21774 | from IPython.utils.coloransi import TermColors, InputTermColors ,ColorScheme, ColorSchemeTable | ||
|
r13366 | from IPython.utils.py3compat import PY3 | ||
|
r22109 | from .colorable import Colorable | ||
|
r13366 | if PY3: | ||
from io import StringIO | ||||
else: | ||||
from StringIO import StringIO | ||||
|
r0 | |||
############################################################################# | ||||
### Python Source Parser (does Hilighting) | ||||
############################################################################# | ||||
_KEYWORD = token.NT_OFFSET + 1 | ||||
_TEXT = token.NT_OFFSET + 2 | ||||
#**************************************************************************** | ||||
# Builtin color schemes | ||||
Colors = TermColors # just a shorthand | ||||
# Build a few color schemes | ||||
NoColor = ColorScheme( | ||||
'NoColor',{ | ||||
|
r21774 | 'header' : Colors.NoColor, | ||
|
r0 | token.NUMBER : Colors.NoColor, | ||
token.OP : Colors.NoColor, | ||||
token.STRING : Colors.NoColor, | ||||
tokenize.COMMENT : Colors.NoColor, | ||||
token.NAME : Colors.NoColor, | ||||
token.ERRORTOKEN : Colors.NoColor, | ||||
_KEYWORD : Colors.NoColor, | ||||
_TEXT : Colors.NoColor, | ||||
|
r21778 | 'in_prompt' : InputTermColors.NoColor, # Input prompt | ||
'in_number' : InputTermColors.NoColor, # Input prompt number | ||||
'in_prompt2' : InputTermColors.NoColor, # Continuation prompt | ||||
|
r21828 | 'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal) | ||
|
r21778 | |||
|
r21828 | 'out_prompt' : Colors.NoColor, # Output prompt | ||
'out_number' : Colors.NoColor, # Output prompt number | ||||
|
r21778 | |||
|
r21828 | 'normal' : Colors.NoColor # color off (usu. Colors.Normal) | ||
|
r0 | } ) | ||
LinuxColors = ColorScheme( | ||||
'Linux',{ | ||||
|
r21774 | 'header' : Colors.LightRed, | ||
|
r0 | token.NUMBER : Colors.LightCyan, | ||
token.OP : Colors.Yellow, | ||||
token.STRING : Colors.LightBlue, | ||||
tokenize.COMMENT : Colors.LightRed, | ||||
|
r4243 | token.NAME : Colors.Normal, | ||
|
r0 | token.ERRORTOKEN : Colors.Red, | ||
_KEYWORD : Colors.LightGreen, | ||||
_TEXT : Colors.Yellow, | ||||
|
r21778 | 'in_prompt' : InputTermColors.Green, | ||
'in_number' : InputTermColors.LightGreen, | ||||
'in_prompt2' : InputTermColors.Green, | ||||
|
r21828 | 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) | ||
|
r21778 | |||
|
r21828 | 'out_prompt' : Colors.Red, | ||
'out_number' : Colors.LightRed, | ||||
|
r21778 | |||
|
r21828 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) | ||
|
r0 | } ) | ||
|
r22609 | NeutralColors = ColorScheme( | ||
'Neutral',{ | ||||
'header' : Colors.Red, | ||||
token.NUMBER : Colors.Cyan, | ||||
token.OP : Colors.Blue, | ||||
token.STRING : Colors.Blue, | ||||
tokenize.COMMENT : Colors.Red, | ||||
token.NAME : Colors.Normal, | ||||
token.ERRORTOKEN : Colors.Red, | ||||
_KEYWORD : Colors.Green, | ||||
_TEXT : Colors.Blue, | ||||
'in_prompt' : InputTermColors.Blue, | ||||
'in_number' : InputTermColors.LightBlue, | ||||
'in_prompt2' : InputTermColors.Blue, | ||||
'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) | ||||
'out_prompt' : Colors.Red, | ||||
'out_number' : Colors.LightRed, | ||||
'normal' : Colors.Normal # color off (usu. Colors.Normal) | ||||
} ) | ||||
|
r0 | LightBGColors = ColorScheme( | ||
'LightBG',{ | ||||
|
r21774 | 'header' : Colors.Red, | ||
|
r0 | token.NUMBER : Colors.Cyan, | ||
token.OP : Colors.Blue, | ||||
token.STRING : Colors.Blue, | ||||
tokenize.COMMENT : Colors.Red, | ||||
|
r4243 | token.NAME : Colors.Normal, | ||
|
r0 | token.ERRORTOKEN : Colors.Red, | ||
|
r22609 | |||
|
r0 | _KEYWORD : Colors.Green, | ||
_TEXT : Colors.Blue, | ||||
|
r21778 | 'in_prompt' : InputTermColors.Blue, | ||
'in_number' : InputTermColors.LightBlue, | ||||
'in_prompt2' : InputTermColors.Blue, | ||||
|
r21828 | 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) | ||
|
r21778 | |||
|
r21828 | 'out_prompt' : Colors.Red, | ||
'out_number' : Colors.LightRed, | ||||
|
r21778 | |||
|
r21828 | 'normal' : Colors.Normal # color off (usu. Colors.Normal) | ||
|
r0 | } ) | ||
# Build table of color schemes (needed by the parser) | ||||
|
r22609 | ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors], | ||
|
r0 | _scheme_default) | ||
|
r22109 | class Parser(Colorable): | ||
|
r0 | """ Format colored Python source. | ||
""" | ||||
|
r22109 | def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None): | ||
|
r0 | """ Create a parser with a specified color table and output channel. | ||
Call format() to process code. | ||||
""" | ||||
|
r22109 | |||
super(Parser, self).__init__(parent=parent) | ||||
|
r0 | self.color_table = color_table and color_table or ANSICodeColors | ||
self.out = out | ||||
def format(self, raw, out = None, scheme = ''): | ||||
|
r553 | return self.format2(raw, out, scheme)[0] | ||
def format2(self, raw, out = None, scheme = ''): | ||||
|
r0 | """ Parse and send the colored source. | ||
If out and scheme are not specified, the defaults (given to | ||||
constructor) are used. | ||||
out should be a file-type object. Optionally, out can be given as the | ||||
string 'str' and the parser will automatically return the output in a | ||||
string.""" | ||||
|
r6001 | |||
|
r0 | string_output = 0 | ||
|
r622 | if out == 'str' or self.out == 'str' or \ | ||
|
r13366 | isinstance(self.out,StringIO): | ||
|
r622 | # XXX - I don't really like this state handling logic, but at this | ||
# point I don't want to make major changes, so adding the | ||||
# isinstance() check is the simplest I can do to ensure correct | ||||
# behavior. | ||||
|
r0 | out_old = self.out | ||
|
r13366 | self.out = StringIO() | ||
|
r0 | string_output = 1 | ||
elif out is not None: | ||||
self.out = out | ||||
|
r588 | |||
# Fast return of the unmodified input for NoColor scheme | ||||
if scheme == 'NoColor': | ||||
error = False | ||||
self.out.write(raw) | ||||
if string_output: | ||||
return raw,error | ||||
else: | ||||
return None,error | ||||
|
r6001 | |||
|
r588 | # local shorthands | ||
|
r0 | colors = self.color_table[scheme].colors | ||
self.colors = colors # put in object so __call__ sees it | ||||
|
r588 | |||
# Remove trailing whitespace and normalize tabs | ||||
self.raw = raw.expandtabs().rstrip() | ||||
|
r6001 | |||
|
r0 | # store line offsets in self.lines | ||
self.lines = [0, 0] | ||||
pos = 0 | ||||
|
r595 | raw_find = self.raw.find | ||
|
r588 | lines_append = self.lines.append | ||
|
r0 | while 1: | ||
|
r588 | pos = raw_find('\n', pos) + 1 | ||
|
r0 | if not pos: break | ||
|
r588 | lines_append(pos) | ||
lines_append(len(self.raw)) | ||||
|
r0 | |||
# parse the source and write it | ||||
self.pos = 0 | ||||
|
r13366 | text = StringIO(self.raw) | ||
|
r553 | |||
error = False | ||||
|
r0 | try: | ||
|
r4768 | for atoken in generate_tokens(text.readline): | ||
self(*atoken) | ||||
except tokenize.TokenError as ex: | ||||
msg = ex.args[0] | ||||
line = ex.args[1][0] | ||||
|
r0 | self.out.write("%s\n\n*** ERROR: %s%s%s\n" % | ||
(colors[token.ERRORTOKEN], | ||||
msg, self.raw[self.lines[line]:], | ||||
colors.normal) | ||||
) | ||||
|
r553 | error = True | ||
|
r0 | self.out.write(colors.normal+'\n') | ||
if string_output: | ||||
output = self.out.getvalue() | ||||
self.out = out_old | ||||
|
r553 | return (output, error) | ||
return (None, error) | ||||
|
r0 | |||
|
r7882 | def __call__(self, toktype, toktext, start_pos, end_pos, line): | ||
|
r0 | """ Token handler, with syntax highlighting.""" | ||
|
r7882 | (srow,scol) = start_pos | ||
(erow,ecol) = end_pos | ||||
|
r0 | colors = self.colors | ||
|
r588 | owrite = self.out.write | ||
|
r0 | |||
# line separator, so this works across platforms | ||||
linesep = os.linesep | ||||
# calculate new positions | ||||
oldpos = self.pos | ||||
newpos = self.lines[srow] + scol | ||||
self.pos = newpos + len(toktext) | ||||
# send the original whitespace, if needed | ||||
if newpos > oldpos: | ||||
|
r588 | owrite(self.raw[oldpos:newpos]) | ||
|
r0 | |||
# skip indenting tokens | ||||
if toktype in [token.INDENT, token.DEDENT]: | ||||
self.pos = newpos | ||||
return | ||||
# map token type to a color group | ||||
|
r21780 | if token.LPAR <= toktype <= token.OP: | ||
|
r0 | toktype = token.OP | ||
elif toktype == token.NAME and keyword.iskeyword(toktext): | ||||
toktype = _KEYWORD | ||||
color = colors.get(toktype, colors[_TEXT]) | ||||
#print '<%s>' % toktext, # dbg | ||||
# Triple quoted strings must be handled carefully so that backtracking | ||||
# in pagers works correctly. We need color terminators on _each_ line. | ||||
if linesep in toktext: | ||||
toktext = toktext.replace(linesep, '%s%s%s' % | ||||
(colors.normal,linesep,color)) | ||||
# send text | ||||
|
r588 | owrite('%s%s%s' % (color,toktext,colors.normal)) | ||
|
r6001 | |||
|
r645 | def main(argv=None): | ||
|
r646 | """Run as a command-line script: colorize a python file or stdin using ANSI | ||
color escapes and print to stdout. | ||||
|
r0 | |||
|
r645 | Inputs: | ||
|
r0 | |||
|
r645 | - argv(None): a list of strings like sys.argv[1:] giving the command-line | ||
arguments. If None, use sys.argv[1:]. | ||||
""" | ||||
|
r0 | |||
|
r646 | usage_msg = """%prog [options] [filename] | ||
|
r0 | |||
|
r646 | Colorize a python file or stdin using ANSI color escapes and print to stdout. | ||
If no filename is given, or if filename is -, read standard input.""" | ||||
|
r645 | |||
|
r9397 | import optparse | ||
|
r645 | parser = optparse.OptionParser(usage=usage_msg) | ||
newopt = parser.add_option | ||||
newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store', | ||||
choices=['Linux','LightBG','NoColor'],default=_scheme_default, | ||||
help="give the color scheme to use. Currently only 'Linux'\ | ||||
(default) and 'LightBG' and 'NoColor' are implemented (give without\ | ||||
quotes)") | ||||
opts,args = parser.parse_args(argv) | ||||
|
r646 | if len(args) > 1: | ||
parser.error("you must give at most one filename.") | ||||
if len(args) == 0: | ||||
fname = '-' # no filename given; setup to read from stdin | ||||
else: | ||||
fname = args[0] | ||||
if fname == '-': | ||||
stream = sys.stdin | ||||
else: | ||||
|
r3232 | try: | ||
|
r6650 | stream = open(fname) | ||
|
r7787 | except IOError as msg: | ||
|
r7817 | print(msg, file=sys.stderr) | ||
|
r3232 | sys.exit(1) | ||
|
r0 | |||
parser = Parser() | ||||
|
r646 | |||
# we need nested try blocks because pre-2.5 python doesn't support unified | ||||
# try-except-finally | ||||
|
r0 | try: | ||
|
r646 | try: | ||
# write colorized version to stdout | ||||
parser.format(stream.read(),scheme=opts.scheme_name) | ||||
|
r7787 | except IOError as msg: | ||
|
r646 | # if user reads through a pager and quits, don't print traceback | ||
if msg.args != (32,'Broken pipe'): | ||||
raise | ||||
finally: | ||||
if stream is not sys.stdin: | ||||
stream.close() # in case a non-handled exception happened above | ||||
|
r0 | |||
if __name__ == "__main__": | ||||
main() | ||||