minirst.py
450 lines
| 15.1 KiB
| text/x-python
|
PythonLexer
/ mercurial / minirst.py
|
r9156 | # minirst.py - minimal reStructuredText parser | ||
# | ||||
|
r10443 | # Copyright 2009, 2010 Matt Mackall <mpm@selenic.com> and others | ||
|
r9156 | # | ||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r9156 | |||
"""simplified reStructuredText parser. | ||||
This parser knows just enough about reStructuredText to parse the | ||||
Mercurial docstrings. | ||||
It cheats in a major way: nested blocks are not really nested. They | ||||
are just indented blocks that look like they are nested. This relies | ||||
on the user to keep the right indentation for the blocks. | ||||
It only supports a small subset of reStructuredText: | ||||
|
r9741 | - sections | ||
|
r9156 | - paragraphs | ||
|
r9741 | - literal blocks | ||
- definition lists | ||||
|
r9156 | |||
|
r12388 | - specific admonitions | ||
|
r9741 | - bullet lists (items must start with '-') | ||
- enumerated lists (no autonumbering) | ||||
|
r9156 | |||
|
r9293 | - field lists (colons cannot be escaped) | ||
|
r9156 | - option lists (supports only long options without arguments) | ||
|
r9741 | - inline literals (no other inline markup is not recognized) | ||
|
r9156 | """ | ||
|
r11297 | import re, sys | ||
|
r11464 | import util, encoding | ||
|
r12388 | from i18n import _ | ||
|
r11464 | |||
def replace(text, substs): | ||||
utext = text.decode(encoding.encoding) | ||||
for f, t in substs: | ||||
utext = utext.replace(f, t) | ||||
return utext.encode(encoding.encoding) | ||||
|
r9156 | |||
|
r12651 | |||
_blockre = re.compile(r"\n(?:\s*\n)+") | ||||
|
r9156 | def findblocks(text): | ||
"""Find continuous blocks of lines in text. | ||||
Returns a list of dictionaries representing the blocks. Each block | ||||
has an 'indent' field and a 'lines' field. | ||||
""" | ||||
|
r12651 | blocks = [] | ||
for b in _blockre.split(text.strip()): | ||||
lines = b.splitlines() | ||||
indent = min((len(l) - len(l.lstrip())) for l in lines) | ||||
lines = [l[indent:] for l in lines] | ||||
blocks.append(dict(indent=indent, lines=lines)) | ||||
|
r9156 | return blocks | ||
def findliteralblocks(blocks): | ||||
"""Finds literal blocks and adds a 'type' field to the blocks. | ||||
Literal blocks are given the type 'literal', all other blocks are | ||||
given type the 'paragraph'. | ||||
""" | ||||
i = 0 | ||||
while i < len(blocks): | ||||
# Searching for a block that looks like this: | ||||
# | ||||
# +------------------------------+ | ||||
# | paragraph | | ||||
# | (ends with "::") | | ||||
# +------------------------------+ | ||||
# +---------------------------+ | ||||
# | indented literal block | | ||||
# +---------------------------+ | ||||
blocks[i]['type'] = 'paragraph' | ||||
|
r10282 | if blocks[i]['lines'][-1].endswith('::') and i + 1 < len(blocks): | ||
|
r9156 | indent = blocks[i]['indent'] | ||
|
r10282 | adjustment = blocks[i + 1]['indent'] - indent | ||
|
r9156 | |||
if blocks[i]['lines'] == ['::']: | ||||
# Expanded form: remove block | ||||
del blocks[i] | ||||
i -= 1 | ||||
elif blocks[i]['lines'][-1].endswith(' ::'): | ||||
# Partially minimized form: remove space and both | ||||
# colons. | ||||
blocks[i]['lines'][-1] = blocks[i]['lines'][-1][:-3] | ||||
else: | ||||
# Fully minimized form: remove just one colon. | ||||
blocks[i]['lines'][-1] = blocks[i]['lines'][-1][:-1] | ||||
# List items are formatted with a hanging indent. We must | ||||
# correct for this here while we still have the original | ||||
# information on the indentation of the subsequent literal | ||||
# blocks available. | ||||
|
r9738 | m = _bulletre.match(blocks[i]['lines'][0]) | ||
if m: | ||||
indent += m.end() | ||||
adjustment -= m.end() | ||||
|
r9156 | |||
# Mark the following indented blocks. | ||||
|
r10282 | while i + 1 < len(blocks) and blocks[i + 1]['indent'] > indent: | ||
blocks[i + 1]['type'] = 'literal' | ||||
blocks[i + 1]['indent'] -= adjustment | ||||
|
r9156 | i += 1 | ||
i += 1 | ||||
return blocks | ||||
|
r10447 | _bulletre = re.compile(r'(-|[0-9A-Za-z]+\.|\(?[0-9A-Za-z]+\)|\|) ') | ||
|
r9737 | _optionre = re.compile(r'^(--[a-z-]+)((?:[ =][a-zA-Z][\w-]*)? +)(.*)$') | ||
|
r10065 | _fieldre = re.compile(r':(?![: ])([^:]*)(?<! ):[ ]+(.*)') | ||
|
r9737 | _definitionre = re.compile(r'[^ ]') | ||
def splitparagraphs(blocks): | ||||
"""Split paragraphs into lists.""" | ||||
# Tuples with (list type, item regexp, single line items?). Order | ||||
# matters: definition lists has the least specific regexp and must | ||||
# come last. | ||||
listtypes = [('bullet', _bulletre, True), | ||||
('option', _optionre, True), | ||||
('field', _fieldre, True), | ||||
('definition', _definitionre, False)] | ||||
def match(lines, i, itemre, singleline): | ||||
"""Does itemre match an item at line i? | ||||
A list item can be followed by an idented line or another list | ||||
item (but only if singleline is True). | ||||
""" | ||||
line1 = lines[i] | ||||
|
r10282 | line2 = i + 1 < len(lines) and lines[i + 1] or '' | ||
|
r9737 | if not itemre.match(line1): | ||
return False | ||||
if singleline: | ||||
return line2 == '' or line2[0] == ' ' or itemre.match(line2) | ||||
else: | ||||
return line2.startswith(' ') | ||||
i = 0 | ||||
while i < len(blocks): | ||||
if blocks[i]['type'] == 'paragraph': | ||||
lines = blocks[i]['lines'] | ||||
for type, itemre, singleline in listtypes: | ||||
if match(lines, 0, itemre, singleline): | ||||
items = [] | ||||
for j, line in enumerate(lines): | ||||
if match(lines, j, itemre, singleline): | ||||
items.append(dict(type=type, lines=[], | ||||
indent=blocks[i]['indent'])) | ||||
items[-1]['lines'].append(line) | ||||
|
r10282 | blocks[i:i + 1] = items | ||
|
r9737 | break | ||
i += 1 | ||||
return blocks | ||||
|
r9156 | |||
|
r10065 | _fieldwidth = 12 | ||
def updatefieldlists(blocks): | ||||
"""Find key and maximum key width for field lists.""" | ||||
i = 0 | ||||
while i < len(blocks): | ||||
if blocks[i]['type'] != 'field': | ||||
i += 1 | ||||
continue | ||||
keywidth = 0 | ||||
j = i | ||||
while j < len(blocks) and blocks[j]['type'] == 'field': | ||||
m = _fieldre.match(blocks[j]['lines'][0]) | ||||
key, rest = m.groups() | ||||
blocks[j]['lines'][0] = rest | ||||
blocks[j]['key'] = key | ||||
keywidth = max(keywidth, len(key)) | ||||
j += 1 | ||||
for block in blocks[i:j]: | ||||
block['keywidth'] = keywidth | ||||
i = j + 1 | ||||
return blocks | ||||
|
r10443 | def prunecontainers(blocks, keep): | ||
"""Prune unwanted containers. | ||||
The blocks must have a 'type' field, i.e., they should have been | ||||
run through findliteralblocks first. | ||||
""" | ||||
|
r10444 | pruned = [] | ||
|
r10443 | i = 0 | ||
while i + 1 < len(blocks): | ||||
# Searching for a block that looks like this: | ||||
# | ||||
# +-------+---------------------------+ | ||||
# | ".. container ::" type | | ||||
# +---+ | | ||||
# | blocks | | ||||
# +-------------------------------+ | ||||
if (blocks[i]['type'] == 'paragraph' and | ||||
blocks[i]['lines'][0].startswith('.. container::')): | ||||
indent = blocks[i]['indent'] | ||||
adjustment = blocks[i + 1]['indent'] - indent | ||||
containertype = blocks[i]['lines'][0][15:] | ||||
prune = containertype not in keep | ||||
|
r10444 | if prune: | ||
pruned.append(containertype) | ||||
|
r10443 | |||
# Always delete "..container:: type" block | ||||
del blocks[i] | ||||
j = i | ||||
while j < len(blocks) and blocks[j]['indent'] > indent: | ||||
if prune: | ||||
del blocks[j] | ||||
i -= 1 # adjust outer index | ||||
else: | ||||
blocks[j]['indent'] -= adjustment | ||||
j += 1 | ||||
i += 1 | ||||
|
r10444 | return blocks, pruned | ||
|
r10443 | |||
|
r10984 | _sectionre = re.compile(r"""^([-=`:.'"~^_*+#])\1+$""") | ||
|
r9156 | def findsections(blocks): | ||
"""Finds sections. | ||||
The blocks must have a 'type' field, i.e., they should have been | ||||
run through findliteralblocks first. | ||||
""" | ||||
for block in blocks: | ||||
# Searching for a block that looks like this: | ||||
# | ||||
# +------------------------------+ | ||||
# | Section title | | ||||
# | ------------- | | ||||
# +------------------------------+ | ||||
if (block['type'] == 'paragraph' and | ||||
len(block['lines']) == 2 and | ||||
|
r10984 | len(block['lines'][0]) == len(block['lines'][1]) and | ||
_sectionre.match(block['lines'][1])): | ||||
|
r10983 | block['underline'] = block['lines'][1][0] | ||
|
r9156 | block['type'] = 'section' | ||
|
r10983 | del block['lines'][1] | ||
|
r9156 | return blocks | ||
|
r9623 | def inlineliterals(blocks): | ||
|
r11464 | substs = [('``', '"')] | ||
|
r9623 | for b in blocks: | ||
|
r10983 | if b['type'] in ('paragraph', 'section'): | ||
|
r11464 | b['lines'] = [replace(l, substs) for l in b['lines']] | ||
|
r9623 | return blocks | ||
|
r10972 | def hgrole(blocks): | ||
|
r11464 | substs = [(':hg:`', '"hg '), ('`', '"')] | ||
|
r10972 | for b in blocks: | ||
|
r10983 | if b['type'] in ('paragraph', 'section'): | ||
|
r11192 | # Turn :hg:`command` into "hg command". This also works | ||
# when there is a line break in the command and relies on | ||||
# the fact that we have no stray back-quotes in the input | ||||
# (run the blocks through inlineliterals first). | ||||
|
r11464 | b['lines'] = [replace(l, substs) for l in b['lines']] | ||
|
r10972 | return blocks | ||
|
r9156 | def addmargins(blocks): | ||
"""Adds empty blocks for vertical spacing. | ||||
This groups bullets, options, and definitions together with no vertical | ||||
space between them, and adds an empty block between all other blocks. | ||||
""" | ||||
i = 1 | ||||
while i < len(blocks): | ||||
|
r10282 | if (blocks[i]['type'] == blocks[i - 1]['type'] and | ||
|
r10936 | blocks[i]['type'] in ('bullet', 'option', 'field')): | ||
|
r9156 | i += 1 | ||
else: | ||||
blocks.insert(i, dict(lines=[''], indent=0, type='margin')) | ||||
i += 2 | ||||
return blocks | ||||
|
r12819 | def prunecomments(blocks): | ||
"""Remove comments.""" | ||||
i = 0 | ||||
while i < len(blocks): | ||||
b = blocks[i] | ||||
if b['type'] == 'paragraph' and b['lines'][0].startswith('.. '): | ||||
del blocks[i] | ||||
else: | ||||
i += 1 | ||||
return blocks | ||||
|
r12620 | _admonitionre = re.compile(r"\.\. (admonition|attention|caution|danger|" | ||
r"error|hint|important|note|tip|warning)::", | ||||
flags=re.IGNORECASE) | ||||
|
r12388 | def findadmonitions(blocks): | ||
""" | ||||
Makes the type of the block an admonition block if | ||||
the first line is an admonition directive | ||||
""" | ||||
i = 0 | ||||
while i < len(blocks): | ||||
|
r12620 | m = _admonitionre.match(blocks[i]['lines'][0]) | ||
|
r12388 | if m: | ||
blocks[i]['type'] = 'admonition' | ||||
admonitiontitle = blocks[i]['lines'][0][3:m.end() - 2].lower() | ||||
firstline = blocks[i]['lines'][0][m.end() + 1:] | ||||
|
r12620 | if firstline: | ||
blocks[i]['lines'].insert(1, ' ' + firstline) | ||||
|
r12388 | |||
blocks[i]['admonitiontitle'] = admonitiontitle | ||||
del blocks[i]['lines'][0] | ||||
i = i + 1 | ||||
return blocks | ||||
|
r9156 | |||
|
r12652 | _admonitiontitles = {'attention': _('Attention:'), | ||
'caution': _('Caution:'), | ||||
'danger': _('!Danger!') , | ||||
'error': _('Error:'), | ||||
'hint': _('Hint:'), | ||||
'important': _('Important:'), | ||||
'note': _('Note:'), | ||||
'tip': _('Tip:'), | ||||
'warning': _('Warning!')} | ||||
|
r9156 | def formatblock(block, width): | ||
"""Format a block according to width.""" | ||||
|
r9417 | if width <= 0: | ||
width = 78 | ||||
|
r9156 | indent = ' ' * block['indent'] | ||
|
r12388 | if block['type'] == 'admonition': | ||
|
r12652 | admonition = _admonitiontitles[block['admonitiontitle']] | ||
|
r12388 | hang = len(block['lines'][-1]) - len(block['lines'][-1].lstrip()) | ||
defindent = indent + hang * ' ' | ||||
text = ' '.join(map(str.strip, block['lines'])) | ||||
return '%s\n%s' % (indent + admonition, util.wrap(text, width=width, | ||||
initindent=defindent, | ||||
hangindent=defindent)) | ||||
|
r9156 | if block['type'] == 'margin': | ||
return '' | ||||
|
r9735 | if block['type'] == 'literal': | ||
|
r9291 | indent += ' ' | ||
return indent + ('\n' + indent).join(block['lines']) | ||||
|
r9735 | if block['type'] == 'section': | ||
|
r10983 | underline = len(block['lines'][0]) * block['underline'] | ||
return "%s%s\n%s%s" % (indent, block['lines'][0],indent, underline) | ||||
|
r9735 | if block['type'] == 'definition': | ||
|
r9156 | term = indent + block['lines'][0] | ||
|
r9737 | hang = len(block['lines'][-1]) - len(block['lines'][-1].lstrip()) | ||
defindent = indent + hang * ' ' | ||||
|
r9156 | text = ' '.join(map(str.strip, block['lines'][1:])) | ||
|
r11297 | return '%s\n%s' % (term, util.wrap(text, width=width, | ||
initindent=defindent, | ||||
hangindent=defindent)) | ||||
|
r10937 | subindent = indent | ||
|
r9735 | if block['type'] == 'bullet': | ||
|
r10447 | if block['lines'][0].startswith('| '): | ||
# Remove bullet for line blocks and add no extra | ||||
# indention. | ||||
block['lines'][0] = block['lines'][0][2:] | ||||
else: | ||||
m = _bulletre.match(block['lines'][0]) | ||||
subindent = indent + m.end() * ' ' | ||||
|
r9737 | elif block['type'] == 'field': | ||
|
r10065 | keywidth = block['keywidth'] | ||
key = block['key'] | ||||
subindent = indent + _fieldwidth * ' ' | ||||
if len(key) + 2 > _fieldwidth: | ||||
# key too large, use full line width | ||||
key = key.ljust(width) | ||||
elif keywidth + 2 < _fieldwidth: | ||||
# all keys are small, add only two spaces | ||||
key = key.ljust(keywidth + 2) | ||||
subindent = indent + (keywidth + 2) * ' ' | ||||
else: | ||||
# mixed sizes, use fieldwidth for this one | ||||
key = key.ljust(_fieldwidth) | ||||
block['lines'][0] = key + block['lines'][0] | ||||
|
r9737 | elif block['type'] == 'option': | ||
m = _optionre.match(block['lines'][0]) | ||||
|
r10064 | option, arg, rest = m.groups() | ||
subindent = indent + (len(option) + len(arg)) * ' ' | ||||
|
r9156 | |||
|
r9737 | text = ' '.join(map(str.strip, block['lines'])) | ||
|
r11297 | return util.wrap(text, width=width, | ||
initindent=indent, | ||||
hangindent=subindent) | ||||
|
r9156 | |||
|
r10444 | def format(text, width, indent=0, keep=None): | ||
|
r9156 | """Parse and format the text according to width.""" | ||
blocks = findblocks(text) | ||||
|
r9540 | for b in blocks: | ||
b['indent'] += indent | ||||
|
r9156 | blocks = findliteralblocks(blocks) | ||
|
r10444 | blocks, pruned = prunecontainers(blocks, keep or []) | ||
|
r10983 | blocks = findsections(blocks) | ||
|
r9623 | blocks = inlineliterals(blocks) | ||
|
r10972 | blocks = hgrole(blocks) | ||
|
r9737 | blocks = splitparagraphs(blocks) | ||
|
r10065 | blocks = updatefieldlists(blocks) | ||
|
r12819 | blocks = prunecomments(blocks) | ||
|
r9156 | blocks = addmargins(blocks) | ||
|
r12388 | blocks = findadmonitions(blocks) | ||
|
r10444 | text = '\n'.join(formatblock(b, width) for b in blocks) | ||
if keep is None: | ||||
return text | ||||
else: | ||||
return text, pruned | ||||
|
r9156 | |||
if __name__ == "__main__": | ||||
from pprint import pprint | ||||
|
r10443 | def debug(func, *args): | ||
blocks = func(*args) | ||||
|
r9156 | print "*** after %s:" % func.__name__ | ||
pprint(blocks) | ||||
return blocks | ||||
text = open(sys.argv[1]).read() | ||||
blocks = debug(findblocks, text) | ||||
blocks = debug(findliteralblocks, blocks) | ||||
|
r11187 | blocks, pruned = debug(prunecontainers, blocks, sys.argv[2:]) | ||
|
r10063 | blocks = debug(inlineliterals, blocks) | ||
|
r9737 | blocks = debug(splitparagraphs, blocks) | ||
|
r10065 | blocks = debug(updatefieldlists, blocks) | ||
|
r9156 | blocks = debug(findsections, blocks) | ||
|
r12819 | blocks = debug(prunecomments, blocks) | ||
|
r9156 | blocks = debug(addmargins, blocks) | ||
|
r12388 | blocks = debug(findadmonitions, blocks) | ||
|
r9156 | print '\n'.join(formatblock(b, 30) for b in blocks) | ||