##// END OF EJS Templates
hgweb: expose input stream on parsed WSGI request object...
hgweb: expose input stream on parsed WSGI request object Our next step towards moving away from wsgirequest to our newer, friendlier parsedrequest type is input stream access. This commit exposes the input stream on the instance. Consumers in the HTTP protocol server switch to it. Because there were very few consumers of the input stream, we stopped storing a reference to the input stream on wsgirequest directly. All access now goes through parsedrequest. However, wsgirequest still may read from this stream as part of cgi.parse(). So we still need to create the stream from wsgirequest. Differential Revision: https://phab.mercurial-scm.org/D2771

File last commit:

r34139:a8994d08 default
r36873:da4e2f87 default
Show More
parser.py
701 lines | 25.4 KiB | text/x-python | PythonLexer
Matt Mackall
revset: introduce basic parser
r11274 # parser.py - simple top-down operator precedence parser for mercurial
#
# Copyright 2010 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Julian Cowley
parser: fix URL to effbot
r11449 # see http://effbot.org/zone/simple-top-down-parsing.htm and
Matt Mackall
revset: introduce basic parser
r11274 # http://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing/
# for background
# takes a tokenizer and elements
Yuya Nishihara
parser: update documentation about tokenizer and elements
r25655 # tokenizer is an iterator that returns (type, value, pos) tuples
Yuya Nishihara
parser: separate actions for primary expression and prefix operator...
r25815 # elements is a mapping of types to binding strength, primary, prefix, infix
# and suffix actions
Matt Mackall
revset: introduce basic parser
r11274 # an action is a tree node name, a tree label, and an optional match
timeless@mozdev.org
en-us: labeled
r17500 # __call__(program) parses program into a labeled tree
Matt Mackall
revset: introduce basic parser
r11274
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 from __future__ import absolute_import, print_function
Gregory Szorc
parser: use absolute_import
r25963
from .i18n import _
Yuya Nishihara
util: wrap s.decode('string_escape') calls for future py3 compatibility
r31484 from . import (
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 encoding,
Yuya Nishihara
util: wrap s.decode('string_escape') calls for future py3 compatibility
r31484 error,
Yuya Nishihara
py3: byte-stringify ValueError of unescapestr() to reraise as ParseError
r36565 pycompat,
Yuya Nishihara
util: wrap s.decode('string_escape') calls for future py3 compatibility
r31484 util,
)
Matt Mackall
revset: raise ParseError exceptions
r11289
Matt Mackall
revset: introduce basic parser
r11274 class parser(object):
Yuya Nishihara
parser: accept iterator of tokens instead of tokenizer function and program...
r25654 def __init__(self, elements, methods=None):
Matt Mackall
revset: introduce basic parser
r11274 self._elements = elements
self._methods = methods
Matt Mackall
templater: use the parser.py parser to extend the templater syntax
r13176 self.current = None
Matt Mackall
revset: introduce basic parser
r11274 def _advance(self):
'advance the tokenizer'
t = self.current
Pierre-Yves David
parsers: use 'next' instead of try/except...
r25171 self.current = next(self._iter, None)
Matt Mackall
revset: introduce basic parser
r11274 return t
Yuya Nishihara
parser: extract function that tests if next token may start new term...
r25804 def _hasnewterm(self):
'True if next token may start new term'
Yuya Nishihara
parser: separate actions for primary expression and prefix operator...
r25815 return any(self._elements[self.current[0]][1:3])
Yuya Nishihara
parser: remove unused parameter 'pos' from _match()...
r25802 def _match(self, m):
Matt Mackall
revset: introduce basic parser
r11274 'make sure the tokenizer matches an end condition'
if self.current[0] != m:
Mads Kiilerich
parsers: fix localization markup of parser errors
r14701 raise error.ParseError(_("unexpected token: %s") % self.current[0],
Dirkjan Ochtman
cleanups: undefined variables
r11305 self.current[2])
Matt Mackall
revset: introduce basic parser
r11274 self._advance()
Yuya Nishihara
parser: factor out function that parses right-hand side of prefix/infix ops...
r25803 def _parseoperand(self, bind, m=None):
'gather right-hand-side operand until an end condition or binding met'
if m and self.current[0] == m:
expr = None
else:
expr = self._parse(bind)
if m:
self._match(m)
return expr
Matt Mackall
revset: introduce basic parser
r11274 def _parse(self, bind=0):
Matt Mackall
revset: raise ParseError exceptions
r11289 token, value, pos = self._advance()
Yuya Nishihara
parser: resolve ambiguity where both prefix and primary actions are defined...
r25816 # handle prefix rules on current token, take as primary if unambiguous
Yuya Nishihara
parser: separate actions for primary expression and prefix operator...
r25815 primary, prefix = self._elements[token][1:3]
Yuya Nishihara
parser: resolve ambiguity where both prefix and primary actions are defined...
r25816 if primary and not (prefix and self._hasnewterm()):
Yuya Nishihara
parser: separate actions for primary expression and prefix operator...
r25815 expr = (primary, value)
elif prefix:
expr = (prefix[0], self._parseoperand(*prefix[1:]))
else:
Mads Kiilerich
parsers: fix localization markup of parser errors
r14701 raise error.ParseError(_("not a prefix: %s") % token, pos)
Matt Mackall
revset: introduce basic parser
r11274 # gather tokens until we meet a lower binding strength
while bind < self._elements[self.current[0]][0]:
Matt Mackall
revset: raise ParseError exceptions
r11289 token, value, pos = self._advance()
Yuya Nishihara
parser: reorder infix/suffix handling to be similar to prefix/primary flow...
r25817 # handle infix rules, take as suffix if unambiguous
Yuya Nishihara
parser: separate actions for primary expression and prefix operator...
r25815 infix, suffix = self._elements[token][3:]
Yuya Nishihara
parser: take suffix action if no infix action is defined...
r25818 if suffix and not (infix and self._hasnewterm()):
Yuya Nishihara
parser: remove unused binding parameter from suffix action...
r29767 expr = (suffix, expr)
Yuya Nishihara
parser: reorder infix/suffix handling to be similar to prefix/primary flow...
r25817 elif infix:
expr = (infix[0], expr, self._parseoperand(*infix[1:]))
Matt Mackall
revset: introduce basic parser
r11274 else:
Yuya Nishihara
parser: reorder infix/suffix handling to be similar to prefix/primary flow...
r25817 raise error.ParseError(_("not an infix: %s") % token, pos)
Matt Mackall
revset: introduce basic parser
r11274 return expr
Yuya Nishihara
parser: accept iterator of tokens instead of tokenizer function and program...
r25654 def parse(self, tokeniter):
'generate a parse tree from tokens'
self._iter = tokeniter
Matt Mackall
templater: use the parser.py parser to extend the templater syntax
r13176 self._advance()
Bernhard Leiner
revset: report a parse error if a revset is not parsed completely (issue2654)
r13665 res = self._parse()
token, value, pos = self.current
return res, pos
Matt Mackall
revset: introduce basic parser
r11274 def eval(self, tree):
'recursively evaluate a parse tree using node methods'
if not isinstance(tree, tuple):
return tree
return self._methods[tree[0]](*[self.eval(t) for t in tree[1:]])
Yuya Nishihara
parser: accept iterator of tokens instead of tokenizer function and program...
r25654 def __call__(self, tokeniter):
'parse tokens into a parse tree and evaluate if methods given'
t = self.parse(tokeniter)
Matt Mackall
revset: introduce basic parser
r11274 if self._methods:
return self.eval(t)
return t
Yuya Nishihara
parser: move prettyformat() function from revset module...
r25253
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 def splitargspec(spec):
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 """Parse spec of function arguments into (poskeys, varkey, keys, optkey)
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> splitargspec(b'')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 ([], None, [], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> splitargspec(b'foo bar')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 ([], None, ['foo', 'bar'], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> splitargspec(b'foo *bar baz **qux')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 (['foo'], 'bar', ['baz'], 'qux')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> splitargspec(b'*foo')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 ([], 'foo', [], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> splitargspec(b'**foo')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 ([], None, [], 'foo')
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 """
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 optkey = None
pre, sep, post = spec.partition('**')
if sep:
posts = post.split()
if not posts:
raise error.ProgrammingError('no **optkey name provided')
if len(posts) > 1:
raise error.ProgrammingError('excessive **optkey names provided')
optkey = posts[0]
pre, sep, post = pre.partition('*')
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 pres = pre.split()
posts = post.split()
if sep:
if not posts:
raise error.ProgrammingError('no *varkey name provided')
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 return pres, posts[0], posts[1:], optkey
return [], None, pres, optkey
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753
def buildargsdict(trees, funcname, argspec, keyvaluenode, keynode):
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 """Build dict from list containing positional and keyword arguments
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 Arguments are specified by a tuple of ``(poskeys, varkey, keys, optkey)``
where
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753
- ``poskeys``: list of names of positional arguments
- ``varkey``: optional argument name that takes up remainder
- ``keys``: list of names that can be either positional or keyword arguments
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 - ``optkey``: optional argument name that takes up excess keyword arguments
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753
If ``varkey`` specified, all ``keys`` must be given as keyword arguments.
Invalid keywords, too few positional arguments, or too many positional
arguments are rejected, but missing keyword arguments are just omitted.
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 """
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 poskeys, varkey, keys, optkey = argspec
Yuya Nishihara
parser: make buildargsdict() precompute position where keyword args start...
r30752 kwstart = next((i for i, x in enumerate(trees) if x[0] == keyvaluenode),
len(trees))
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 if kwstart < len(poskeys):
raise error.ParseError(_("%(func)s takes at least %(nargs)d positional "
"arguments")
% {'func': funcname, 'nargs': len(poskeys)})
Yuya Nishihara
parser: verify excessive number of args excluding kwargs in buildargsdict()...
r31920 if not varkey and kwstart > len(poskeys) + len(keys):
raise error.ParseError(_("%(func)s takes at most %(nargs)d positional "
"arguments")
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 % {'func': funcname,
'nargs': len(poskeys) + len(keys)})
Yuya Nishihara
parser: preserve order of keyword arguments...
r31922 args = util.sortdict()
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 # consume positional arguments
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 for k, x in zip(poskeys, trees[:kwstart]):
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 args[k] = x
Yuya Nishihara
parser: extend buildargsdict() to support variable-length positional args...
r30753 if varkey:
args[varkey] = trees[len(args):kwstart]
else:
for k, x in zip(keys, trees[len(args):kwstart]):
args[k] = x
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 # remainder should be keyword arguments
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 if optkey:
Yuya Nishihara
parser: preserve order of keyword arguments...
r31922 args[optkey] = util.sortdict()
Yuya Nishihara
parser: make buildargsdict() precompute position where keyword args start...
r30752 for x in trees[kwstart:]:
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 if x[0] != keyvaluenode or x[1][0] != keynode:
raise error.ParseError(_("%(func)s got an invalid argument")
% {'func': funcname})
k = x[1][1]
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 if k in keys:
d = args
elif not optkey:
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 raise error.ParseError(_("%(func)s got an unexpected keyword "
"argument '%(key)s'")
% {'func': funcname, 'key': k})
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 else:
d = args[optkey]
if k in d:
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 raise error.ParseError(_("%(func)s got multiple values for keyword "
"argument '%(key)s'")
% {'func': funcname, 'key': k})
Yuya Nishihara
parser: extend buildargsdict() to support arbitrary number of **kwargs...
r31921 d[k] = x[2]
Yuya Nishihara
revset: add function to build dict of positional and keyword arguments...
r25705 return args
Yuya Nishihara
parser: move unescape helper from templater...
r26231 def unescapestr(s):
try:
Yuya Nishihara
util: wrap s.decode('string_escape') calls for future py3 compatibility
r31484 return util.unescapestr(s)
Yuya Nishihara
parser: move unescape helper from templater...
r26231 except ValueError as e:
# mangle Python's exception into our format
Yuya Nishihara
py3: byte-stringify ValueError of unescapestr() to reraise as ParseError
r36565 raise error.ParseError(pycompat.bytestr(e).lower())
Yuya Nishihara
parser: move unescape helper from templater...
r26231
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 def _brepr(obj):
if isinstance(obj, bytes):
return b"'%s'" % util.escapestr(obj)
return encoding.strtolocal(repr(obj))
Yuya Nishihara
parser: extract closure of prettyformat() to a top-level function...
r25254 def _prettyformat(tree, leafnodes, level, lines):
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 if not isinstance(tree, tuple):
lines.append((level, _brepr(tree)))
elif tree[0] in leafnodes:
rs = map(_brepr, tree[1:])
lines.append((level, '(%s %s)' % (tree[0], ' '.join(rs))))
Yuya Nishihara
parser: extract closure of prettyformat() to a top-level function...
r25254 else:
lines.append((level, '(%s' % tree[0]))
for s in tree[1:]:
_prettyformat(s, leafnodes, level + 1, lines)
lines[-1:] = [(lines[-1][0], lines[-1][1] + ')')]
Yuya Nishihara
parser: move prettyformat() function from revset module...
r25253 def prettyformat(tree, leafnodes):
lines = []
Yuya Nishihara
parser: extract closure of prettyformat() to a top-level function...
r25254 _prettyformat(tree, leafnodes, 0, lines)
Yuya Nishihara
parser: move prettyformat() function from revset module...
r25253 output = '\n'.join((' ' * l + s) for l, s in lines)
return output
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306
def simplifyinfixops(tree, targetnodes):
"""Flatten chained infix operations to reduce usage of Python stack
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 >>> from . import pycompat
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 >>> def f(tree):
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 ... s = prettyformat(simplifyinfixops(tree, (b'or',)), (b'symbol',))
... print(pycompat.sysstr(s))
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'or',
... (b'or',
... (b'symbol', b'1'),
... (b'symbol', b'2')),
... (b'symbol', b'3')))
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol '1')
(symbol '2')
(symbol '3'))
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'func',
... (b'symbol', b'p1'),
... (b'or',
... (b'or',
... (b'func',
... (b'symbol', b'sort'),
... (b'list',
... (b'or',
... (b'or',
... (b'symbol', b'1'),
... (b'symbol', b'2')),
... (b'symbol', b'3')),
... (b'negate',
... (b'symbol', b'rev')))),
... (b'and',
... (b'symbol', b'4'),
... (b'group',
... (b'or',
... (b'or',
... (b'symbol', b'5'),
... (b'symbol', b'6')),
... (b'symbol', b'7'))))),
... (b'symbol', b'8'))))
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (func
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol 'p1')
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (or
(func
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol 'sort')
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (list
(or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol '1')
(symbol '2')
(symbol '3'))
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (negate
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol 'rev'))))
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (and
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol '4')
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 (group
(or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (symbol '5')
(symbol '6')
(symbol '7'))))
(symbol '8')))
Yuya Nishihara
parser: add helper to reduce nesting of chained infix operations...
r25306 """
if not isinstance(tree, tuple):
return tree
op = tree[0]
if op not in targetnodes:
return (op,) + tuple(simplifyinfixops(x, targetnodes) for x in tree[1:])
# walk down left nodes taking each right node. no recursion to left nodes
# because infix operators are left-associative, i.e. left tree is deep.
# e.g. '1 + 2 + 3' -> (+ (+ 1 2) 3) -> (+ 1 2 3)
simplified = []
x = tree
while x[0] == op:
l, r = x[1:]
simplified.append(simplifyinfixops(r, targetnodes))
x = l
simplified.append(simplifyinfixops(x, targetnodes))
simplified.append(op)
return tuple(reversed(simplified))
Yuya Nishihara
parser: move parsererrordetail() function from revset module...
r28720
Yuya Nishihara
parser: add helper function that constructs parsed tree from template...
r34045 def _buildtree(template, placeholder, replstack):
if template == placeholder:
return replstack.pop()
if not isinstance(template, tuple):
return template
return tuple(_buildtree(x, placeholder, replstack) for x in template)
def buildtree(template, placeholder, *repls):
"""Create new tree by substituting placeholders by replacements
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> _ = (b'symbol', b'_')
Yuya Nishihara
parser: add helper function that constructs parsed tree from template...
r34045 >>> def f(template, *repls):
... return buildtree(template, _, *repls)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'func', (b'symbol', b'only'), (b'list', _, _)),
Yuya Nishihara
parser: add helper function that constructs parsed tree from template...
r34045 ... ('symbol', '1'), ('symbol', '2'))
('func', ('symbol', 'only'), ('list', ('symbol', '1'), ('symbol', '2')))
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'and', _, (b'not', _)), (b'symbol', b'1'), (b'symbol', b'2'))
Yuya Nishihara
parser: add helper function that constructs parsed tree from template...
r34045 ('and', ('symbol', '1'), ('not', ('symbol', '2')))
"""
if not isinstance(placeholder, tuple):
raise error.ProgrammingError('placeholder must be a node tuple')
replstack = list(reversed(repls))
r = _buildtree(template, placeholder, replstack)
if replstack:
raise error.ProgrammingError('too many replacements')
return r
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047 def _matchtree(pattern, tree, placeholder, incompletenodes, matches):
if pattern == tree:
return True
if not isinstance(pattern, tuple) or not isinstance(tree, tuple):
return False
if pattern == placeholder and tree[0] not in incompletenodes:
matches.append(tree)
return True
if len(pattern) != len(tree):
return False
return all(_matchtree(p, x, placeholder, incompletenodes, matches)
for p, x in zip(pattern, tree))
def matchtree(pattern, tree, placeholder=None, incompletenodes=()):
"""If a tree matches the pattern, return a list of the tree and nodes
matched with the placeholder; Otherwise None
>>> def f(pattern, tree):
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... m = matchtree(pattern, tree, _, {b'keyvalue', b'list'})
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047 ... if m:
... return m[1:]
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> _ = (b'symbol', b'_')
>>> f((b'func', (b'symbol', b'ancestors'), _),
... (b'func', (b'symbol', b'ancestors'), (b'symbol', b'1')))
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047 [('symbol', '1')]
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'func', (b'symbol', b'ancestors'), _),
... (b'func', (b'symbol', b'ancestors'), None))
>>> f((b'range', (b'dagrange', _, _), _),
... (b'range',
... (b'dagrange', (b'symbol', b'1'), (b'symbol', b'2')),
... (b'symbol', b'3')))
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047 [('symbol', '1'), ('symbol', '2'), ('symbol', '3')]
The placeholder does not match the specified incomplete nodes because
an incomplete node (e.g. argument list) cannot construct an expression.
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'func', (b'symbol', b'ancestors'), _),
... (b'func', (b'symbol', b'ancestors'),
... (b'list', (b'symbol', b'1'), (b'symbol', b'2'))))
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047
The placeholder may be omitted, but which shouldn't match a None node.
>>> _ = None
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> f((b'func', (b'symbol', b'ancestors'), None),
... (b'func', (b'symbol', b'ancestors'), (b'symbol', b'0')))
Yuya Nishihara
parser: add helper function to test if pattern matches parsed tree...
r34047 """
if placeholder is not None and not isinstance(placeholder, tuple):
raise error.ProgrammingError('placeholder must be a node tuple')
matches = [tree]
if _matchtree(pattern, tree, placeholder, incompletenodes, matches):
return matches
Yuya Nishihara
parser: move parsererrordetail() function from revset module...
r28720 def parseerrordetail(inst):
"""Compose error message from specified ParseError object
"""
if len(inst.args) > 1:
Augie Fackler
parser: use %d instead of %s for interpolating error position...
r31353 return _('at %d: %s') % (inst.args[1], inst.args[0])
Yuya Nishihara
parser: move parsererrordetail() function from revset module...
r28720 else:
return inst.args[0]
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 class alias(object):
"""Parsed result of alias"""
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 def __init__(self, name, args, err, replacement):
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 self.name = name
self.args = args
self.error = err
self.replacement = replacement
# whether own `error` information is already shown or not.
Yuya Nishihara
revset: rename findaliases() to expandaliases()...
r28898 # this avoids showing same warning multiple times at each
# `expandaliases`.
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 self.warned = False
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870 class basealiasrules(object):
"""Parsing and expansion rule set of aliases
This is a helper for fileset/revset/template aliases. A concrete rule set
should be made by sub-classing this and implementing class/static methods.
Mads Kiilerich
spelling: fixes of non-dictionary words
r30332 It supports alias expansion of symbol and function-call styles::
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870
# decl = defn
h = heads(default)
b($1) = ancestors($1) - ancestors(default)
"""
# typically a config section, which will be included in error messages
_section = None
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 # tag of symbol node
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870 _symbolnode = 'symbol'
def __new__(cls):
raise TypeError("'%s' is not instantiatable" % cls.__name__)
@staticmethod
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 def _parse(spec):
"""Parse an alias name, arguments and definition"""
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870 raise NotImplementedError
@staticmethod
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 def _trygetfunc(tree):
"""Return (name, args) if tree is a function; otherwise None"""
Yuya Nishihara
parser: add stub class that will host alias parsing and expansion...
r28870 raise NotImplementedError
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
@classmethod
def _builddecl(cls, decl):
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 """Parse an alias declaration into ``(name, args, errorstr)``
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
This function analyzes the parsed tree. The parsing rule is provided
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 by ``_parse()``.
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
- ``name``: of declared alias (may be ``decl`` itself at error)
- ``args``: list of argument names (or None for symbol declaration)
- ``errorstr``: detail about detected error (or None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> sym = lambda x: (b'symbol', x)
>>> symlist = lambda *xs: (b'list',) + tuple(sym(x) for x in xs)
>>> func = lambda n, a: (b'func', sym(n), a)
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 >>> parsemap = {
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... b'foo': sym(b'foo'),
... b'$foo': sym(b'$foo'),
... b'foo::bar': (b'dagrange', sym(b'foo'), sym(b'bar')),
... b'foo()': func(b'foo', None),
... b'$foo()': func(b'$foo', None),
... b'foo($1, $2)': func(b'foo', symlist(b'$1', b'$2')),
... b'foo(bar_bar, baz.baz)':
... func(b'foo', symlist(b'bar_bar', b'baz.baz')),
... b'foo(bar($1, $2))':
... func(b'foo', func(b'bar', symlist(b'$1', b'$2'))),
... b'foo($1, $2, nested($1, $2))':
... func(b'foo', (symlist(b'$1', b'$2') +
... (func(b'nested', symlist(b'$1', b'$2')),))),
... b'foo("bar")': func(b'foo', (b'string', b'bar')),
... b'foo($1, $2': error.ParseError(b'unexpected token: end', 10),
... b'foo("bar': error.ParseError(b'unterminated string', 5),
... b'foo($1, $2, $1)': func(b'foo', symlist(b'$1', b'$2', b'$1')),
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 ... }
>>> def parse(expr):
... x = parsemap[expr]
... if isinstance(x, Exception):
... raise x
... return x
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 >>> def trygetfunc(tree):
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... if not tree or tree[0] != b'func' or tree[1][0] != b'symbol':
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 ... return None
... if not tree[2]:
... return tree[1][1], []
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... if tree[2][0] == b'list':
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 ... return tree[1][1], list(tree[2][1:])
... return tree[1][1], [tree[2]]
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 >>> class aliasrules(basealiasrules):
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 ... _parse = staticmethod(parse)
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 ... _trygetfunc = staticmethod(trygetfunc)
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 >>> builddecl = aliasrules._builddecl
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo', None, None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'$foo')
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 ('$foo', None, "invalid symbol '$foo'")
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo::bar')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo::bar', None, 'invalid format')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo()')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo', [], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'$foo()')
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 ('$foo()', None, "invalid function '$foo'")
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo($1, $2)')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo', ['$1', '$2'], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo(bar_bar, baz.baz)')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo', ['bar_bar', 'baz.baz'], None)
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo($1, $2, nested($1, $2))')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo($1, $2, nested($1, $2))', None, 'invalid argument list')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo(bar($1, $2))')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo(bar($1, $2))', None, 'invalid argument list')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo("bar")')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo("bar")', None, 'invalid argument list')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo($1, $2')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo($1, $2', None, 'at 10: unexpected token: end')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo("bar')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo("bar', None, 'at 5: unterminated string')
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> builddecl(b'foo($1, $2, $1)')
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 ('foo', None, 'argument names collide with each other')
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 """
try:
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 tree = cls._parse(decl)
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 except error.ParseError as inst:
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 return (decl, None, parseerrordetail(inst))
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
if tree[0] == cls._symbolnode:
# "name = ...." style
name = tree[1]
if name.startswith('$'):
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 return (decl, None, _("invalid symbol '%s'") % name)
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 return (name, None, None)
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 func = cls._trygetfunc(tree)
if func:
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 # "name(arg, ....) = ...." style
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 name, args = func
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 if name.startswith('$'):
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 return (decl, None, _("invalid function '%s'") % name)
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 if any(t[0] != cls._symbolnode for t in args):
return (decl, None, _("invalid argument list"))
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871 if len(args) != len(set(args)):
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 return (name, None, _("argument names collide with each other"))
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 return (name, [t[1] for t in args], None)
Yuya Nishihara
parser: move alias declaration parser to common rule-set class...
r28871
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 return (decl, None, _("invalid format"))
Yuya Nishihara
parser: move _relabelaliasargs() to common rule-set class...
r28872
@classmethod
def _relabelargs(cls, tree, args):
"""Mark alias arguments as ``_aliasarg``"""
if not isinstance(tree, tuple):
return tree
op = tree[0]
if op != cls._symbolnode:
return (op,) + tuple(cls._relabelargs(x, args) for x in tree[1:])
assert len(tree) == 2
sym = tree[1]
if sym in args:
op = '_aliasarg'
elif sym.startswith('$'):
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 raise error.ParseError(_("invalid symbol '%s'") % sym)
Yuya Nishihara
parser: move _relabelaliasargs() to common rule-set class...
r28872 return (op, sym)
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873
@classmethod
def _builddefn(cls, defn, args):
"""Parse an alias definition into a tree and marks substitutions
This function marks alias argument references as ``_aliasarg``. The
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 parsing rule is provided by ``_parse()``.
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873
``args`` is a list of alias argument names, or None if the alias
is declared as a symbol.
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 >>> from . import pycompat
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 >>> parsemap = {
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... b'$1 or foo': (b'or', (b'symbol', b'$1'), (b'symbol', b'foo')),
... b'$1 or $bar':
... (b'or', (b'symbol', b'$1'), (b'symbol', b'$bar')),
... b'$10 or baz':
... (b'or', (b'symbol', b'$10'), (b'symbol', b'baz')),
... b'"$1" or "foo"':
... (b'or', (b'string', b'$1'), (b'string', b'foo')),
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 ... }
>>> class aliasrules(basealiasrules):
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 ... _parse = staticmethod(parsemap.__getitem__)
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 ... _trygetfunc = staticmethod(lambda x: None)
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 >>> builddefn = aliasrules._builddefn
>>> def pprint(tree):
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 ... s = prettyformat(tree, (b'_aliasarg', b'string', b'symbol'))
... print(pycompat.sysstr(s))
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> args = [b'$1', b'$2', b'foo']
>>> pprint(builddefn(b'$1 or foo', args))
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 (or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (_aliasarg '$1')
(_aliasarg 'foo'))
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 >>> try:
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 ... builddefn(b'$1 or $bar', args)
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 ... except error.ParseError as inst:
Yuya Nishihara
doctest: use print_function and convert bytes to unicode where needed
r34139 ... print(pycompat.sysstr(parseerrordetail(inst)))
Yuya Nishihara
parser: rephrase "'$' not for alias arguments" message...
r29058 invalid symbol '$bar'
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> args = [b'$1', b'$10', b'foo']
>>> pprint(builddefn(b'$10 or baz', args))
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 (or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (_aliasarg '$10')
(symbol 'baz'))
Yuya Nishihara
doctest: bulk-replace string literals with b'' for Python 3...
r34133 >>> pprint(builddefn(b'"$1" or "foo"', args))
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 (or
Yuya Nishihara
parser: stabilize output of prettyformat() by using byte-safe repr()...
r34075 (string '$1')
(string 'foo'))
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 """
Yuya Nishihara
parser: unify parser function of alias declaration and definition...
r28875 tree = cls._parse(defn)
Yuya Nishihara
parser: move alias definition parser to common rule-set class...
r28873 if args:
args = set(args)
else:
args = set()
return cls._relabelargs(tree, args)
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892
@classmethod
def build(cls, decl, defn):
"""Parse an alias declaration and definition into an alias object"""
repl = efmt = None
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 name, args, err = cls._builddecl(decl)
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 if err:
Yuya Nishihara
parser: shorten prefix of alias parsing errors...
r29059 efmt = _('bad declaration of %(section)s "%(name)s": %(error)s')
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 else:
try:
repl = cls._builddefn(defn, args)
except error.ParseError as inst:
err = parseerrordetail(inst)
Yuya Nishihara
parser: shorten prefix of alias parsing errors...
r29059 efmt = _('bad definition of %(section)s "%(name)s": %(error)s')
Yuya Nishihara
parser: construct alias object by rule-set class...
r28892 if err:
err = efmt % {'section': cls._section, 'name': name, 'error': err}
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 return alias(name, args, err, repl)
Yuya Nishihara
parser: extract helper that creates a dict of aliases...
r28893
@classmethod
def buildmap(cls, items):
"""Parse a list of alias (name, replacement) pairs into a dict of
alias objects"""
aliases = {}
for decl, defn in items:
a = cls.build(decl, defn)
aliases[a.name] = a
return aliases
Yuya Nishihara
parser: move functions that process alias expansion to rule-set class...
r28895
@classmethod
def _getalias(cls, aliases, tree):
Yuya Nishihara
parser: make _getalias() return (alias, pattern-args) pair...
r28909 """If tree looks like an unexpanded alias, return (alias, pattern-args)
pair. Return None otherwise.
Yuya Nishihara
parser: move functions that process alias expansion to rule-set class...
r28895 """
if not isinstance(tree, tuple):
return None
if tree[0] == cls._symbolnode:
name = tree[1]
a = aliases.get(name)
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 if a and a.args is None:
Yuya Nishihara
parser: make _getalias() return (alias, pattern-args) pair...
r28909 return a, None
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 func = cls._trygetfunc(tree)
if func:
name, args = func
Yuya Nishihara
parser: move functions that process alias expansion to rule-set class...
r28895 a = aliases.get(name)
Yuya Nishihara
parser: drop redundant comparison between alias declaration tree and pattern...
r28908 if a and a.args is not None:
Yuya Nishihara
parser: factor out _trygetfunc() that extracts function name and arguments...
r28910 return a, args
Yuya Nishihara
parser: move functions that process alias expansion to rule-set class...
r28895 return None
@classmethod
def _expandargs(cls, tree, args):
"""Replace _aliasarg instances with the substitution value of the
same name in args, recursively.
"""
if not isinstance(tree, tuple):
return tree
if tree[0] == '_aliasarg':
sym = tree[1]
return args[sym]
return tuple(cls._expandargs(t, args) for t in tree)
@classmethod
def _expand(cls, aliases, tree, expanding, cache):
if not isinstance(tree, tuple):
return tree
Yuya Nishihara
parser: make _getalias() return (alias, pattern-args) pair...
r28909 r = cls._getalias(aliases, tree)
if r is None:
Yuya Nishihara
parser: reorder alias expansion routine to return early...
r28896 return tuple(cls._expand(aliases, t, expanding, cache)
for t in tree)
Yuya Nishihara
parser: make _getalias() return (alias, pattern-args) pair...
r28909 a, l = r
Yuya Nishihara
parser: reorder alias expansion routine to return early...
r28896 if a.error:
raise error.Abort(a.error)
if a in expanding:
raise error.ParseError(_('infinite expansion of %(section)s '
'"%(name)s" detected')
% {'section': cls._section, 'name': a.name})
Yuya Nishihara
parser: add short comment how aliases are expanded in phases
r28897 # get cacheable replacement tree by expanding aliases recursively
Yuya Nishihara
parser: reorder alias expansion routine to return early...
r28896 expanding.append(a)
if a.name not in cache:
cache[a.name] = cls._expand(aliases, a.replacement, expanding,
cache)
result = cache[a.name]
expanding.pop()
if a.args is None:
return result
Yuya Nishihara
parser: add short comment how aliases are expanded in phases
r28897 # substitute function arguments in replacement tree
Yuya Nishihara
parser: reorder alias expansion routine to return early...
r28896 if len(l) != len(a.args):
raise error.ParseError(_('invalid number of arguments: %d')
% len(l))
l = [cls._expand(aliases, t, [], cache) for t in l]
return cls._expandargs(result, dict(zip(a.args, l)))
Yuya Nishihara
parser: move functions that process alias expansion to rule-set class...
r28895
@classmethod
def expand(cls, aliases, tree):
"""Expand aliases in tree, recursively.
'aliases' is a dictionary mapping user defined aliases to alias objects.
"""
return cls._expand(aliases, tree, [], {})