|
|
# -*- coding: utf-8 -*-
|
|
|
"""
|
|
|
Class and program to colorize python source code for ANSI terminals.
|
|
|
|
|
|
Based on an HTML code highlighter by Jurgen Hermann found at:
|
|
|
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
|
|
|
|
|
|
Modifications by Fernando Perez (fperez@colorado.edu).
|
|
|
|
|
|
Information on the original HTML highlighter follows:
|
|
|
|
|
|
MoinMoin - Python Source Parser
|
|
|
|
|
|
Title: Colorize Python source using the built-in tokenizer
|
|
|
|
|
|
Submitter: Jurgen Hermann
|
|
|
Last Updated:2001/04/06
|
|
|
|
|
|
Version no:1.2
|
|
|
|
|
|
Description:
|
|
|
|
|
|
This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
|
|
|
Python source code to HTML markup, rendering comments, keywords,
|
|
|
operators, numeric and string literals in different colors.
|
|
|
|
|
|
It shows how to use the built-in keyword, token and tokenize modules to
|
|
|
scan Python source code and re-emit it with no changes to its original
|
|
|
formatting (which is the hard part).
|
|
|
"""
|
|
|
|
|
|
__all__ = ['ANSICodeColors', 'Parser']
|
|
|
|
|
|
_scheme_default = 'Linux'
|
|
|
|
|
|
|
|
|
# Imports
|
|
|
import keyword
|
|
|
import os
|
|
|
import sys
|
|
|
import token
|
|
|
import tokenize
|
|
|
|
|
|
generate_tokens = tokenize.generate_tokens
|
|
|
|
|
|
from IPython.utils.coloransi import TermColors, InputTermColors,ColorScheme, ColorSchemeTable
|
|
|
from .colorable import Colorable
|
|
|
from io import StringIO
|
|
|
|
|
|
#############################################################################
|
|
|
### Python Source Parser (does Highlighting)
|
|
|
#############################################################################
|
|
|
|
|
|
_KEYWORD = token.NT_OFFSET + 1
|
|
|
_TEXT = token.NT_OFFSET + 2
|
|
|
|
|
|
#****************************************************************************
|
|
|
# Builtin color schemes
|
|
|
|
|
|
Colors = TermColors # just a shorthand
|
|
|
|
|
|
# Build a few color schemes
|
|
|
NoColor = ColorScheme(
|
|
|
'NoColor',{
|
|
|
'header' : Colors.NoColor,
|
|
|
token.NUMBER : Colors.NoColor,
|
|
|
token.OP : Colors.NoColor,
|
|
|
token.STRING : Colors.NoColor,
|
|
|
tokenize.COMMENT : Colors.NoColor,
|
|
|
token.NAME : Colors.NoColor,
|
|
|
token.ERRORTOKEN : Colors.NoColor,
|
|
|
|
|
|
_KEYWORD : Colors.NoColor,
|
|
|
_TEXT : Colors.NoColor,
|
|
|
|
|
|
'in_prompt' : InputTermColors.NoColor, # Input prompt
|
|
|
'in_number' : InputTermColors.NoColor, # Input prompt number
|
|
|
'in_prompt2' : InputTermColors.NoColor, # Continuation prompt
|
|
|
'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal)
|
|
|
|
|
|
'out_prompt' : Colors.NoColor, # Output prompt
|
|
|
'out_number' : Colors.NoColor, # Output prompt number
|
|
|
|
|
|
'normal' : Colors.NoColor # color off (usu. Colors.Normal)
|
|
|
} )
|
|
|
|
|
|
LinuxColors = ColorScheme(
|
|
|
'Linux',{
|
|
|
'header' : Colors.LightRed,
|
|
|
token.NUMBER : Colors.LightCyan,
|
|
|
token.OP : Colors.Yellow,
|
|
|
token.STRING : Colors.LightBlue,
|
|
|
tokenize.COMMENT : Colors.LightRed,
|
|
|
token.NAME : Colors.Normal,
|
|
|
token.ERRORTOKEN : Colors.Red,
|
|
|
|
|
|
_KEYWORD : Colors.LightGreen,
|
|
|
_TEXT : Colors.Yellow,
|
|
|
|
|
|
'in_prompt' : InputTermColors.Green,
|
|
|
'in_number' : InputTermColors.LightGreen,
|
|
|
'in_prompt2' : InputTermColors.Green,
|
|
|
'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
|
|
|
|
|
|
'out_prompt' : Colors.Red,
|
|
|
'out_number' : Colors.LightRed,
|
|
|
|
|
|
'normal' : Colors.Normal # color off (usu. Colors.Normal)
|
|
|
} )
|
|
|
|
|
|
NeutralColors = ColorScheme(
|
|
|
'Neutral',{
|
|
|
'header' : Colors.Red,
|
|
|
token.NUMBER : Colors.Cyan,
|
|
|
token.OP : Colors.Blue,
|
|
|
token.STRING : Colors.Blue,
|
|
|
tokenize.COMMENT : Colors.Red,
|
|
|
token.NAME : Colors.Normal,
|
|
|
token.ERRORTOKEN : Colors.Red,
|
|
|
|
|
|
_KEYWORD : Colors.Green,
|
|
|
_TEXT : Colors.Blue,
|
|
|
|
|
|
'in_prompt' : InputTermColors.Blue,
|
|
|
'in_number' : InputTermColors.LightBlue,
|
|
|
'in_prompt2' : InputTermColors.Blue,
|
|
|
'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
|
|
|
|
|
|
'out_prompt' : Colors.Red,
|
|
|
'out_number' : Colors.LightRed,
|
|
|
|
|
|
'normal' : Colors.Normal # color off (usu. Colors.Normal)
|
|
|
} )
|
|
|
|
|
|
# Hack: the 'neutral' colours are not very visible on a dark background on
|
|
|
# Windows. Since Windows command prompts have a dark background by default, and
|
|
|
# relatively few users are likely to alter that, we will use the 'Linux' colours,
|
|
|
# designed for a dark background, as the default on Windows. Changing it here
|
|
|
# avoids affecting the prompt colours rendered by prompt_toolkit, where the
|
|
|
# neutral defaults do work OK.
|
|
|
|
|
|
if os.name == 'nt':
|
|
|
NeutralColors = LinuxColors.copy(name='Neutral')
|
|
|
|
|
|
LightBGColors = ColorScheme(
|
|
|
'LightBG',{
|
|
|
'header' : Colors.Red,
|
|
|
token.NUMBER : Colors.Cyan,
|
|
|
token.OP : Colors.Blue,
|
|
|
token.STRING : Colors.Blue,
|
|
|
tokenize.COMMENT : Colors.Red,
|
|
|
token.NAME : Colors.Normal,
|
|
|
token.ERRORTOKEN : Colors.Red,
|
|
|
|
|
|
|
|
|
_KEYWORD : Colors.Green,
|
|
|
_TEXT : Colors.Blue,
|
|
|
|
|
|
'in_prompt' : InputTermColors.Blue,
|
|
|
'in_number' : InputTermColors.LightBlue,
|
|
|
'in_prompt2' : InputTermColors.Blue,
|
|
|
'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal)
|
|
|
|
|
|
'out_prompt' : Colors.Red,
|
|
|
'out_number' : Colors.LightRed,
|
|
|
|
|
|
'normal' : Colors.Normal # color off (usu. Colors.Normal)
|
|
|
} )
|
|
|
|
|
|
# Build table of color schemes (needed by the parser)
|
|
|
ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors],
|
|
|
_scheme_default)
|
|
|
|
|
|
Undefined = object()
|
|
|
|
|
|
class Parser(Colorable):
|
|
|
""" Format colored Python source.
|
|
|
"""
|
|
|
|
|
|
def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None):
|
|
|
""" Create a parser with a specified color table and output channel.
|
|
|
|
|
|
Call format() to process code.
|
|
|
"""
|
|
|
|
|
|
super(Parser, self).__init__(parent=parent)
|
|
|
|
|
|
self.color_table = color_table if color_table else ANSICodeColors
|
|
|
self.out = out
|
|
|
self.pos = None
|
|
|
self.lines = None
|
|
|
self.raw = None
|
|
|
if not style:
|
|
|
self.style = self.default_style
|
|
|
else:
|
|
|
self.style = style
|
|
|
|
|
|
|
|
|
def format(self, raw, out=None, scheme=Undefined):
|
|
|
import warnings
|
|
|
if scheme is not Undefined:
|
|
|
warnings.warn('The `scheme` argument of IPython.utils.PyColorize:Parser.format is deprecated since IPython 6.0.'
|
|
|
'It will have no effect. Set the parser `style` directly.',
|
|
|
stacklevel=2)
|
|
|
return self.format2(raw, out)[0]
|
|
|
|
|
|
def format2(self, raw, out = None):
|
|
|
""" Parse and send the colored source.
|
|
|
|
|
|
If out and scheme are not specified, the defaults (given to
|
|
|
constructor) are used.
|
|
|
|
|
|
out should be a file-type object. Optionally, out can be given as the
|
|
|
string 'str' and the parser will automatically return the output in a
|
|
|
string."""
|
|
|
|
|
|
string_output = 0
|
|
|
if out == 'str' or self.out == 'str' or \
|
|
|
isinstance(self.out, StringIO):
|
|
|
# XXX - I don't really like this state handling logic, but at this
|
|
|
# point I don't want to make major changes, so adding the
|
|
|
# isinstance() check is the simplest I can do to ensure correct
|
|
|
# behavior.
|
|
|
out_old = self.out
|
|
|
self.out = StringIO()
|
|
|
string_output = 1
|
|
|
elif out is not None:
|
|
|
self.out = out
|
|
|
else:
|
|
|
raise ValueError('`out` or `self.out` should be file-like or the value `"str"`')
|
|
|
|
|
|
# Fast return of the unmodified input for NoColor scheme
|
|
|
if self.style == 'NoColor':
|
|
|
error = False
|
|
|
self.out.write(raw)
|
|
|
if string_output:
|
|
|
return raw, error
|
|
|
return None, error
|
|
|
|
|
|
# local shorthands
|
|
|
colors = self.color_table[self.style].colors
|
|
|
self.colors = colors # put in object so __call__ sees it
|
|
|
|
|
|
# Remove trailing whitespace and normalize tabs
|
|
|
self.raw = raw.expandtabs().rstrip()
|
|
|
|
|
|
# store line offsets in self.lines
|
|
|
self.lines = [0, 0]
|
|
|
pos = 0
|
|
|
raw_find = self.raw.find
|
|
|
lines_append = self.lines.append
|
|
|
while True:
|
|
|
pos = raw_find('\n', pos) + 1
|
|
|
if not pos:
|
|
|
break
|
|
|
lines_append(pos)
|
|
|
lines_append(len(self.raw))
|
|
|
|
|
|
# parse the source and write it
|
|
|
self.pos = 0
|
|
|
text = StringIO(self.raw)
|
|
|
|
|
|
error = False
|
|
|
try:
|
|
|
for atoken in generate_tokens(text.readline):
|
|
|
self(*atoken)
|
|
|
except tokenize.TokenError as ex:
|
|
|
msg = ex.args[0]
|
|
|
line = ex.args[1][0]
|
|
|
self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
|
|
|
(colors[token.ERRORTOKEN],
|
|
|
msg, self.raw[self.lines[line]:],
|
|
|
colors.normal)
|
|
|
)
|
|
|
error = True
|
|
|
self.out.write(colors.normal+'\n')
|
|
|
if string_output:
|
|
|
output = self.out.getvalue()
|
|
|
self.out = out_old
|
|
|
return (output, error)
|
|
|
return (None, error)
|
|
|
|
|
|
|
|
|
def _inner_call_(self, toktype, toktext, start_pos):
|
|
|
"""like call but write to a temporary buffer"""
|
|
|
buff = StringIO()
|
|
|
srow, scol = start_pos
|
|
|
colors = self.colors
|
|
|
owrite = buff.write
|
|
|
|
|
|
# line separator, so this works across platforms
|
|
|
linesep = os.linesep
|
|
|
|
|
|
# calculate new positions
|
|
|
oldpos = self.pos
|
|
|
newpos = self.lines[srow] + scol
|
|
|
self.pos = newpos + len(toktext)
|
|
|
|
|
|
# send the original whitespace, if needed
|
|
|
if newpos > oldpos:
|
|
|
owrite(self.raw[oldpos:newpos])
|
|
|
|
|
|
# skip indenting tokens
|
|
|
if toktype in [token.INDENT, token.DEDENT]:
|
|
|
self.pos = newpos
|
|
|
buff.seek(0)
|
|
|
return buff.read()
|
|
|
|
|
|
# map token type to a color group
|
|
|
if token.LPAR <= toktype <= token.OP:
|
|
|
toktype = token.OP
|
|
|
elif toktype == token.NAME and keyword.iskeyword(toktext):
|
|
|
toktype = _KEYWORD
|
|
|
color = colors.get(toktype, colors[_TEXT])
|
|
|
|
|
|
# Triple quoted strings must be handled carefully so that backtracking
|
|
|
# in pagers works correctly. We need color terminators on _each_ line.
|
|
|
if linesep in toktext:
|
|
|
toktext = toktext.replace(linesep, '%s%s%s' %
|
|
|
(colors.normal,linesep,color))
|
|
|
|
|
|
# send text
|
|
|
owrite('%s%s%s' % (color,toktext,colors.normal))
|
|
|
buff.seek(0)
|
|
|
return buff.read()
|
|
|
|
|
|
|
|
|
def __call__(self, toktype, toktext, start_pos, end_pos, line):
|
|
|
""" Token handler, with syntax highlighting."""
|
|
|
self.out.write(
|
|
|
self._inner_call_(toktype, toktext, start_pos))
|
|
|
|
