wireproto.py
1056 lines
| 36.5 KiB
| text/x-python
|
PythonLexer
/ mercurial / wireproto.py
|
r11581 | # wireproto.py - generic wire protocol support functions | ||
# | ||||
# Copyright 2005-2010 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r25993 | from __future__ import absolute_import | ||
|
r29341 | import hashlib | ||
|
r25993 | import os | ||
import tempfile | ||||
|
r11581 | |||
|
r25993 | from .i18n import _ | ||
from .node import ( | ||||
bin, | ||||
hex, | ||||
|
r32260 | nullid, | ||
|
r25993 | ) | ||
from . import ( | ||||
bundle2, | ||||
changegroup as changegroupmod, | ||||
|
r34099 | discovery, | ||
|
r25993 | encoding, | ||
error, | ||||
exchange, | ||||
peer, | ||||
pushkey as pushkeymod, | ||||
|
r30924 | pycompat, | ||
|
r33805 | repository, | ||
|
r26443 | streamclone, | ||
|
r25993 | util, | ||
) | ||||
|
r20903 | |||
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
|
r30909 | bundle2requiredmain = _('incompatible Mercurial client; bundle2 required') | ||
bundle2requiredhint = _('see https://www.mercurial-scm.org/wiki/' | ||||
'IncompatibleClient') | ||||
bundle2required = '%s\n(%s)\n' % (bundle2requiredmain, bundle2requiredhint) | ||||
|
r27246 | |||
|
r20903 | class abstractserverproto(object): | ||
"""abstract class that summarizes the protocol API | ||||
Used as reference and documentation. | ||||
""" | ||||
def getargs(self, args): | ||||
"""return the value for arguments in <args> | ||||
returns a list of values (same order as <args>)""" | ||||
raise NotImplementedError() | ||||
def getfile(self, fp): | ||||
"""write the whole content of a file into a file like object | ||||
The file is in the form:: | ||||
(<chunk-size>\n<chunk>)+0\n | ||||
chunk size is the ascii version of the int. | ||||
""" | ||||
raise NotImplementedError() | ||||
def redirect(self): | ||||
"""may setup interception for stdout and stderr | ||||
See also the `restore` method.""" | ||||
raise NotImplementedError() | ||||
# If the `redirect` function does install interception, the `restore` | ||||
# function MUST be defined. If interception is not used, this function | ||||
# MUST NOT be defined. | ||||
# | ||||
# left commented here on purpose | ||||
# | ||||
#def restore(self): | ||||
# """reinstall previous stdout and stderr and return intercepted stdout | ||||
# """ | ||||
# raise NotImplementedError() | ||||
|
r28436 | class remoteiterbatcher(peer.iterbatcher): | ||
def __init__(self, remote): | ||||
super(remoteiterbatcher, self).__init__() | ||||
self._remote = remote | ||||
|
r28438 | def __getattr__(self, name): | ||
|
r33759 | # Validate this method is batchable, since submit() only supports | ||
# batchable methods. | ||||
fn = getattr(self._remote, name) | ||||
if not getattr(fn, 'batchable', None): | ||||
raise error.ProgrammingError('Attempted to batch a non-batchable ' | ||||
'call to %r' % name) | ||||
|
r28438 | return super(remoteiterbatcher, self).__getattr__(name) | ||
|
r28436 | def submit(self): | ||
"""Break the batch request into many patch calls and pipeline them. | ||||
This is mostly valuable over http where request sizes can be | ||||
limited, but can be used in other places as well. | ||||
""" | ||||
|
r33761 | # 2-tuple of (command, arguments) that represents what will be | ||
# sent over the wire. | ||||
requests = [] | ||||
# 4-tuple of (command, final future, @batchable generator, remote | ||||
# future). | ||||
results = [] | ||||
for command, args, opts, finalfuture in self.calls: | ||||
mtd = getattr(self._remote, command) | ||||
|
r34727 | batchable = mtd.batchable(mtd.__self__, *args, **opts) | ||
|
r33761 | |||
commandargs, fremote = next(batchable) | ||||
assert fremote | ||||
requests.append((command, commandargs)) | ||||
results.append((command, finalfuture, batchable, fremote)) | ||||
if requests: | ||||
self._resultiter = self._remote._submitbatch(requests) | ||||
self._results = results | ||||
|
r28436 | |||
def results(self): | ||||
|
r33761 | for command, finalfuture, batchable, remotefuture in self._results: | ||
# Get the raw result, set it in the remote future, feed it | ||||
# back into the @batchable generator so it can be decoded, and | ||||
# set the result on the final future to this value. | ||||
remoteresult = next(self._resultiter) | ||||
remotefuture.set(remoteresult) | ||||
finalfuture.set(next(batchable)) | ||||
# Verify our @batchable generators only emit 2 values. | ||||
try: | ||||
next(batchable) | ||||
except StopIteration: | ||||
pass | ||||
else: | ||||
raise error.ProgrammingError('%s @batchable generator emitted ' | ||||
'unexpected value count' % command) | ||||
yield finalfuture.value | ||||
|
r28436 | |||
|
r25912 | # Forward a couple of names from peer to make wireproto interactions | ||
# slightly more sensible. | ||||
batchable = peer.batchable | ||||
future = peer.future | ||||
|
r14621 | |||
|
r11597 | # list of nodes encoding / decoding | ||
def decodelist(l, sep=' '): | ||||
|
r13722 | if l: | ||
|
r34730 | return [bin(v) for v in l.split(sep)] | ||
|
r13722 | return [] | ||
|
r11597 | |||
def encodelist(l, sep=' '): | ||||
|
r23848 | try: | ||
return sep.join(map(hex, l)) | ||||
except TypeError: | ||||
raise | ||||
|
r11597 | |||
|
r14622 | # batched call argument encoding | ||
def escapearg(plain): | ||||
return (plain | ||||
|
r25708 | .replace(':', ':c') | ||
.replace(',', ':o') | ||||
.replace(';', ':s') | ||||
.replace('=', ':e')) | ||||
|
r14622 | |||
def unescapearg(escaped): | ||||
return (escaped | ||||
|
r25708 | .replace(':e', '=') | ||
.replace(':s', ';') | ||||
.replace(':o', ',') | ||||
.replace(':c', ':')) | ||||
|
r14622 | |||
|
r29733 | def encodebatchcmds(req): | ||
"""Return a ``cmds`` argument value for the ``batch`` command.""" | ||||
cmds = [] | ||||
for op, argsdict in req: | ||||
|
r29734 | # Old servers didn't properly unescape argument names. So prevent | ||
# the sending of argument names that may not be decoded properly by | ||||
# servers. | ||||
assert all(escapearg(k) == k for k in argsdict) | ||||
|
r29733 | args = ','.join('%s=%s' % (escapearg(k), escapearg(v)) | ||
for k, v in argsdict.iteritems()) | ||||
cmds.append('%s %s' % (op, args)) | ||||
return ';'.join(cmds) | ||||
|
r21646 | # mapping of options accepted by getbundle and their types | ||
# | ||||
# Meant to be extended by extensions. It is extensions responsibility to ensure | ||||
# such options are properly processed in exchange.getbundle. | ||||
# | ||||
# supported types are: | ||||
# | ||||
# :nodes: list of binary nodes | ||||
# :csv: list of comma-separated values | ||||
|
r25403 | # :scsv: list of comma-separated values return as set | ||
|
r21646 | # :plain: string with no transformation needed. | ||
gboptsmap = {'heads': 'nodes', | ||||
|
r35268 | 'bookmarks': 'boolean', | ||
|
r21646 | 'common': 'nodes', | ||
|
r22353 | 'obsmarkers': 'boolean', | ||
|
r34323 | 'phases': 'boolean', | ||
|
r25403 | 'bundlecaps': 'scsv', | ||
|
r21989 | 'listkeys': 'csv', | ||
|
r26690 | 'cg': 'boolean', | ||
'cbattempted': 'boolean'} | ||||
|
r21646 | |||
|
r11586 | # client side | ||
|
r33805 | class wirepeer(repository.legacypeer): | ||
|
r27243 | """Client-side interface for communicating with a peer repository. | ||
|
r14622 | |||
|
r27243 | Methods commonly call wire protocol commands of the same name. | ||
See also httppeer.py and sshpeer.py for protocol-specific | ||||
implementations of this interface. | ||||
""" | ||||
|
r33805 | # Begin of basewirepeer interface. | ||
|
r14622 | |||
|
r28436 | def iterbatch(self): | ||
return remoteiterbatcher(self) | ||||
|
r14623 | @batchable | ||
|
r11586 | def lookup(self, key): | ||
self.requirecap('lookup', _('look up remote revision')) | ||||
|
r14623 | f = future() | ||
|
r20671 | yield {'key': encoding.fromlocal(key)}, f | ||
|
r14623 | d = f.value | ||
|
r11586 | success, data = d[:-1].split(" ", 1) | ||
if int(success): | ||||
|
r14623 | yield bin(data) | ||
|
r34064 | else: | ||
self._abort(error.RepoError(data)) | ||||
|
r11586 | |||
|
r14623 | @batchable | ||
|
r11586 | def heads(self): | ||
|
r14623 | f = future() | ||
yield {}, f | ||||
d = f.value | ||||
|
r11586 | try: | ||
|
r14623 | yield decodelist(d[:-1]) | ||
|
r13726 | except ValueError: | ||
|
r11879 | self._abort(error.ResponseError(_("unexpected response:"), d)) | ||
|
r11586 | |||
|
r14623 | @batchable | ||
|
r13723 | def known(self, nodes): | ||
|
r14623 | f = future() | ||
|
r20671 | yield {'nodes': encodelist(nodes)}, f | ||
|
r14623 | d = f.value | ||
|
r13723 | try: | ||
|
r22201 | yield [bool(int(b)) for b in d] | ||
|
r13726 | except ValueError: | ||
|
r13723 | self._abort(error.ResponseError(_("unexpected response:"), d)) | ||
|
r14623 | @batchable | ||
|
r11586 | def branchmap(self): | ||
|
r14623 | f = future() | ||
yield {}, f | ||||
d = f.value | ||||
|
r11586 | try: | ||
branchmap = {} | ||||
for branchpart in d.splitlines(): | ||||
|
r11597 | branchname, branchheads = branchpart.split(' ', 1) | ||
|
r28883 | branchname = encoding.tolocal(urlreq.unquote(branchname)) | ||
|
r11597 | branchheads = decodelist(branchheads) | ||
|
r11586 | branchmap[branchname] = branchheads | ||
|
r14623 | yield branchmap | ||
|
r11586 | except TypeError: | ||
self._abort(error.ResponseError(_("unexpected response:"), d)) | ||||
|
r33805 | @batchable | ||
def listkeys(self, namespace): | ||||
if not self.capable('pushkey'): | ||||
yield {}, None | ||||
f = future() | ||||
self.ui.debug('preparing listkeys for "%s"\n' % namespace) | ||||
yield {'namespace': encoding.fromlocal(namespace)}, f | ||||
d = f.value | ||||
self.ui.debug('received listkey for "%s": %i bytes\n' | ||||
% (namespace, len(d))) | ||||
yield pushkeymod.decodekeys(d) | ||||
|
r11586 | |||
|
r14623 | @batchable | ||
|
r11586 | def pushkey(self, namespace, key, old, new): | ||
if not self.capable('pushkey'): | ||||
|
r14623 | yield False, None | ||
f = future() | ||||
|
r17293 | self.ui.debug('preparing pushkey for "%s:%s"\n' % (namespace, key)) | ||
|
r20671 | yield {'namespace': encoding.fromlocal(namespace), | ||
'key': encoding.fromlocal(key), | ||||
'old': encoding.fromlocal(old), | ||||
'new': encoding.fromlocal(new)}, f | ||||
|
r14623 | d = f.value | ||
|
r15652 | d, output = d.split('\n', 1) | ||
|
r13450 | try: | ||
d = bool(int(d)) | ||||
except ValueError: | ||||
raise error.ResponseError( | ||||
_('push failed (unexpected response):'), d) | ||||
|
r15652 | for l in output.splitlines(True): | ||
self.ui.status(_('remote: '), l) | ||||
|
r14623 | yield d | ||
|
r11586 | |||
|
r11588 | def stream_out(self): | ||
return self._callstream('stream_out') | ||||
|
r21646 | def getbundle(self, source, **kwargs): | ||
|
r34740 | kwargs = pycompat.byteskwargs(kwargs) | ||
|
r13741 | self.requirecap('getbundle', _('look up remote changes')) | ||
opts = {} | ||||
|
r25128 | bundlecaps = kwargs.get('bundlecaps') | ||
if bundlecaps is not None: | ||||
kwargs['bundlecaps'] = sorted(bundlecaps) | ||||
else: | ||||
bundlecaps = () # kwargs could have it to None | ||||
|
r21646 | for key, value in kwargs.iteritems(): | ||
if value is None: | ||||
continue | ||||
keytype = gboptsmap.get(key) | ||||
if keytype is None: | ||||
|
r34731 | raise error.ProgrammingError( | ||
'Unexpectedly None keytype for key %s' % key) | ||||
|
r21646 | elif keytype == 'nodes': | ||
value = encodelist(value) | ||||
|
r25403 | elif keytype in ('csv', 'scsv'): | ||
|
r21646 | value = ','.join(value) | ||
|
r21988 | elif keytype == 'boolean': | ||
|
r22351 | value = '%i' % bool(value) | ||
|
r21646 | elif keytype != 'plain': | ||
raise KeyError('unknown getbundle option type %s' | ||||
% keytype) | ||||
opts[key] = value | ||||
|
r34740 | f = self._callcompressable("getbundle", **pycompat.strkwargs(opts)) | ||
|
r25149 | if any((cap.startswith('HG2') for cap in bundlecaps)): | ||
|
r24641 | return bundle2.getunbundler(self.ui, f) | ||
|
r21069 | else: | ||
|
r22390 | return changegroupmod.cg1unpacker(f, 'UN') | ||
|
r13741 | |||
|
r29706 | def unbundle(self, cg, heads, url): | ||
|
r11592 | '''Send cg (a readable file-like object representing the | ||
changegroup to push, typically a chunkbuffer object) to the | ||||
|
r21075 | remote server as a bundle. | ||
When pushing a bundle10 stream, return an integer indicating the | ||||
|
r32880 | result of the push (see changegroup.apply()). | ||
|
r21075 | |||
|
r29706 | When pushing a bundle20 stream, return a bundle20 stream. | ||
`url` is the url the client thinks it's pushing to, which is | ||||
visible to hooks. | ||||
''' | ||||
|
r11592 | |||
|
r14419 | if heads != ['force'] and self.capable('unbundlehash'): | ||
|
r13942 | heads = encodelist(['hashed', | ||
|
r29341 | hashlib.sha1(''.join(sorted(heads))).digest()]) | ||
|
r13942 | else: | ||
heads = encodelist(heads) | ||||
|
r21075 | if util.safehasattr(cg, 'deltaheader'): | ||
# this a bundle10, do the old style call sequence | ||||
ret, output = self._callpush("unbundle", cg, heads=heads) | ||||
if ret == "": | ||||
raise error.ResponseError( | ||||
_('push failed:'), output) | ||||
try: | ||||
ret = int(ret) | ||||
except ValueError: | ||||
raise error.ResponseError( | ||||
_('push failed (unexpected response):'), ret) | ||||
|
r11592 | |||
|
r21075 | for l in output.splitlines(True): | ||
self.ui.status(_('remote: '), l) | ||||
else: | ||||
# bundle2 push. Send a stream, fetch a stream. | ||||
stream = self._calltwowaystream('unbundle', cg, heads=heads) | ||||
|
r24641 | ret = bundle2.getunbundler(self.ui, stream) | ||
|
r11592 | return ret | ||
|
r33805 | # End of basewirepeer interface. | ||
# Begin of baselegacywirepeer interface. | ||||
def branches(self, nodes): | ||||
n = encodelist(nodes) | ||||
d = self._call("branches", nodes=n) | ||||
try: | ||||
br = [tuple(decodelist(b)) for b in d.splitlines()] | ||||
return br | ||||
except ValueError: | ||||
self._abort(error.ResponseError(_("unexpected response:"), d)) | ||||
def between(self, pairs): | ||||
batch = 8 # avoid giant requests | ||||
r = [] | ||||
for i in xrange(0, len(pairs), batch): | ||||
n = " ".join([encodelist(p, '-') for p in pairs[i:i + batch]]) | ||||
d = self._call("between", pairs=n) | ||||
try: | ||||
r.extend(l and decodelist(l) or [] for l in d.splitlines()) | ||||
except ValueError: | ||||
self._abort(error.ResponseError(_("unexpected response:"), d)) | ||||
return r | ||||
def changegroup(self, nodes, kind): | ||||
n = encodelist(nodes) | ||||
f = self._callcompressable("changegroup", roots=n) | ||||
return changegroupmod.cg1unpacker(f, 'UN') | ||||
def changegroupsubset(self, bases, heads, kind): | ||||
self.requirecap('changegroupsubset', _('look up remote changes')) | ||||
bases = encodelist(bases) | ||||
heads = encodelist(heads) | ||||
f = self._callcompressable("changegroupsubset", | ||||
bases=bases, heads=heads) | ||||
return changegroupmod.cg1unpacker(f, 'UN') | ||||
# End of baselegacywirepeer interface. | ||||
def _submitbatch(self, req): | ||||
"""run batch request <req> on the server | ||||
Returns an iterator of the raw responses from the server. | ||||
""" | ||||
rsp = self._callstream("batch", cmds=encodebatchcmds(req)) | ||||
chunk = rsp.read(1024) | ||||
work = [chunk] | ||||
while chunk: | ||||
while ';' not in chunk and chunk: | ||||
chunk = rsp.read(1024) | ||||
work.append(chunk) | ||||
merged = ''.join(work) | ||||
while ';' in merged: | ||||
one, merged = merged.split(';', 1) | ||||
yield unescapearg(one) | ||||
chunk = rsp.read(1024) | ||||
work = [merged, chunk] | ||||
yield unescapearg(''.join(work)) | ||||
def _submitone(self, op, args): | ||||
|
r34740 | return self._call(op, **pycompat.strkwargs(args)) | ||
|
r33805 | |||
|
r14048 | def debugwireargs(self, one, two, three=None, four=None, five=None): | ||
|
r13720 | # don't pass optional arguments left at their default value | ||
opts = {} | ||||
if three is not None: | ||||
|
r35375 | opts[r'three'] = three | ||
|
r13720 | if four is not None: | ||
|
r35375 | opts[r'four'] = four | ||
|
r13720 | return self._call('debugwireargs', one=one, two=two, **opts) | ||
|
r20904 | def _call(self, cmd, **args): | ||
"""execute <cmd> on the server | ||||
The command is expected to return a simple string. | ||||
returns the server reply as a string.""" | ||||
raise NotImplementedError() | ||||
def _callstream(self, cmd, **args): | ||||
"""execute <cmd> on the server | ||||
|
r28435 | The command is expected to return a stream. Note that if the | ||
command doesn't return a stream, _callstream behaves | ||||
differently for ssh and http peers. | ||||
|
r20904 | |||
|
r28435 | returns the server reply as a file like object. | ||
""" | ||||
|
r20904 | raise NotImplementedError() | ||
|
r20905 | def _callcompressable(self, cmd, **args): | ||
"""execute <cmd> on the server | ||||
The command is expected to return a stream. | ||||
|
r21024 | The stream may have been compressed in some implementations. This | ||
|
r20905 | function takes care of the decompression. This is the only difference | ||
with _callstream. | ||||
returns the server reply as a file like object. | ||||
""" | ||||
raise NotImplementedError() | ||||
|
r20904 | def _callpush(self, cmd, fp, **args): | ||
"""execute a <cmd> on server | ||||
The command is expected to be related to a push. Push has a special | ||||
return method. | ||||
returns the server reply as a (ret, output) tuple. ret is either | ||||
empty (error) or a stringified int. | ||||
""" | ||||
raise NotImplementedError() | ||||
|
r21072 | def _calltwowaystream(self, cmd, fp, **args): | ||
"""execute <cmd> on server | ||||
The command will send a stream to the server and get a stream in reply. | ||||
""" | ||||
raise NotImplementedError() | ||||
|
r20904 | def _abort(self, exception): | ||
"""clearly abort the wire protocol connection and raise the exception | ||||
""" | ||||
raise NotImplementedError() | ||||
|
r11586 | # server side | ||
|
r20902 | # wire protocol command can either return a string or one of these classes. | ||
|
r11625 | class streamres(object): | ||
|
r20902 | """wireproto reply: binary stream | ||
The call was successful and the result is a stream. | ||||
|
r30466 | |||
Accepts either a generator or an object with a ``read(size)`` method. | ||||
``v1compressible`` indicates whether this data can be compressed to | ||||
"version 1" clients (technically: HTTP peers using | ||||
application/mercurial-0.1 media type). This flag should NOT be used on | ||||
new commands because new clients should support a more modern compression | ||||
mechanism. | ||||
|
r20902 | """ | ||
|
r30466 | def __init__(self, gen=None, reader=None, v1compressible=False): | ||
|
r11625 | self.gen = gen | ||
|
r30466 | self.reader = reader | ||
self.v1compressible = v1compressible | ||||
|
r11625 | |||
class pushres(object): | ||||
|
r20902 | """wireproto reply: success with simple integer return | ||
The call was successful and returned an integer contained in `self.res`. | ||||
""" | ||||
|
r11625 | def __init__(self, res): | ||
self.res = res | ||||
|
r12703 | class pusherr(object): | ||
|
r20902 | """wireproto reply: failure | ||
The call failed. The `self.res` attribute contains the error message. | ||||
""" | ||||
|
r12703 | def __init__(self, res): | ||
self.res = res | ||||
|
r15017 | class ooberror(object): | ||
|
r20902 | """wireproto reply: failure of a batch of operation | ||
Something failed during a batch call. The error message is stored in | ||||
`self.message`. | ||||
""" | ||||
|
r15017 | def __init__(self, message): | ||
self.message = message | ||||
|
r29590 | def getdispatchrepo(repo, proto, command): | ||
"""Obtain the repo used for processing wire protocol commands. | ||||
The intent of this function is to serve as a monkeypatch point for | ||||
extensions that need commands to operate on different repo views under | ||||
specialized circumstances. | ||||
""" | ||||
return repo.filtered('served') | ||||
|
r11581 | def dispatch(repo, proto, command): | ||
|
r29590 | repo = getdispatchrepo(repo, proto, command) | ||
|
r11581 | func, spec = commands[command] | ||
args = proto.getargs(spec) | ||||
|
r11625 | return func(repo, proto, *args) | ||
|
r11581 | |||
|
r13721 | def options(cmd, keys, others): | ||
opts = {} | ||||
for k in keys: | ||||
if k in others: | ||||
opts[k] = others[k] | ||||
del others[k] | ||||
if others: | ||||
|
r30473 | util.stderr.write("warning: %s ignored unexpected arguments %s\n" | ||
% (cmd, ",".join(others))) | ||||
|
r13721 | return opts | ||
|
r27633 | def bundle1allowed(repo, action): | ||
"""Whether a bundle1 operation is allowed from the server. | ||||
Priority is: | ||||
1. server.bundle1gd.<action> (if generaldelta active) | ||||
2. server.bundle1.<action> | ||||
3. server.bundle1gd (if generaldelta active) | ||||
4. server.bundle1 | ||||
""" | ||||
ui = repo.ui | ||||
gd = 'generaldelta' in repo.requirements | ||||
if gd: | ||||
|
r34614 | v = ui.configbool('server', 'bundle1gd.%s' % action) | ||
|
r27633 | if v is not None: | ||
return v | ||||
|
r34614 | v = ui.configbool('server', 'bundle1.%s' % action) | ||
|
r27246 | if v is not None: | ||
return v | ||||
|
r27633 | if gd: | ||
r33217 | v = ui.configbool('server', 'bundle1gd') | |||
|
r27633 | if v is not None: | ||
return v | ||||
r33216 | return ui.configbool('server', 'bundle1') | |||
|
r27246 | |||
|
r30762 | def supportedcompengines(ui, proto, role): | ||
"""Obtain the list of supported compression engines for a request.""" | ||||
assert role in (util.CLIENTROLE, util.SERVERROLE) | ||||
compengines = util.compengines.supportedwireengines(role) | ||||
# Allow config to override default list and ordering. | ||||
if role == util.SERVERROLE: | ||||
configengines = ui.configlist('server', 'compressionengines') | ||||
config = 'server.compressionengines' | ||||
else: | ||||
# This is currently implemented mainly to facilitate testing. In most | ||||
# cases, the server should be in charge of choosing a compression engine | ||||
# because a server has the most to lose from a sub-optimal choice. (e.g. | ||||
# CPU DoS due to an expensive engine or a network DoS due to poor | ||||
# compression ratio). | ||||
configengines = ui.configlist('experimental', | ||||
'clientcompressionengines') | ||||
config = 'experimental.clientcompressionengines' | ||||
# No explicit config. Filter out the ones that aren't supposed to be | ||||
# advertised and return default ordering. | ||||
if not configengines: | ||||
attr = 'serverpriority' if role == util.SERVERROLE else 'clientpriority' | ||||
return [e for e in compengines | ||||
if getattr(e.wireprotosupport(), attr) > 0] | ||||
# If compression engines are listed in the config, assume there is a good | ||||
# reason for it (like server operators wanting to achieve specific | ||||
# performance characteristics). So fail fast if the config references | ||||
# unusable compression engines. | ||||
validnames = set(e.name() for e in compengines) | ||||
invalidnames = set(e for e in configengines if e not in validnames) | ||||
if invalidnames: | ||||
raise error.Abort(_('invalid compression engine defined in %s: %s') % | ||||
(config, ', '.join(sorted(invalidnames)))) | ||||
compengines = [e for e in compengines if e.name() in configengines] | ||||
compengines = sorted(compengines, | ||||
key=lambda e: configengines.index(e.name())) | ||||
if not compengines: | ||||
raise error.Abort(_('%s config option does not specify any known ' | ||||
'compression engines') % config, | ||||
hint=_('usable compression engines: %s') % | ||||
', '.sorted(validnames)) | ||||
return compengines | ||||
|
r20906 | # list of commands | ||
commands = {} | ||||
def wireprotocommand(name, args=''): | ||||
|
r21024 | """decorator for wire protocol command""" | ||
|
r20906 | def register(func): | ||
commands[name] = (func, args) | ||||
return func | ||||
return register | ||||
|
r20907 | @wireprotocommand('batch', 'cmds *') | ||
|
r14622 | def batch(repo, proto, cmds, others): | ||
|
r18382 | repo = repo.filtered("served") | ||
|
r14622 | res = [] | ||
for pair in cmds.split(';'): | ||||
op, args = pair.split(' ', 1) | ||||
vals = {} | ||||
for a in args.split(','): | ||||
if a: | ||||
n, v = a.split('=') | ||||
|
r29734 | vals[unescapearg(n)] = unescapearg(v) | ||
|
r14622 | func, spec = commands[op] | ||
if spec: | ||||
keys = spec.split() | ||||
data = {} | ||||
for k in keys: | ||||
if k == '*': | ||||
star = {} | ||||
for key in vals.keys(): | ||||
if key not in keys: | ||||
star[key] = vals[key] | ||||
data['*'] = star | ||||
else: | ||||
data[k] = vals[k] | ||||
result = func(repo, proto, *[data[k] for k in keys]) | ||||
else: | ||||
result = func(repo, proto) | ||||
|
r15017 | if isinstance(result, ooberror): | ||
return result | ||||
|
r14622 | res.append(escapearg(result)) | ||
return ';'.join(res) | ||||
|
r20908 | @wireprotocommand('between', 'pairs') | ||
|
r11583 | def between(repo, proto, pairs): | ||
|
r11597 | pairs = [decodelist(p, '-') for p in pairs.split(" ")] | ||
|
r11581 | r = [] | ||
for b in repo.between(pairs): | ||||
|
r11597 | r.append(encodelist(b) + "\n") | ||
|
r11581 | return "".join(r) | ||
|
r20909 | @wireprotocommand('branchmap') | ||
|
r11583 | def branchmap(repo, proto): | ||
|
r18281 | branchmap = repo.branchmap() | ||
|
r11581 | heads = [] | ||
for branch, nodes in branchmap.iteritems(): | ||||
|
r28883 | branchname = urlreq.quote(encoding.fromlocal(branch)) | ||
|
r11597 | branchnodes = encodelist(nodes) | ||
heads.append('%s %s' % (branchname, branchnodes)) | ||||
|
r11581 | return '\n'.join(heads) | ||
|
r20910 | @wireprotocommand('branches', 'nodes') | ||
|
r11583 | def branches(repo, proto, nodes): | ||
|
r11597 | nodes = decodelist(nodes) | ||
|
r11581 | r = [] | ||
for b in repo.branches(nodes): | ||||
|
r11597 | r.append(encodelist(b) + "\n") | ||
|
r11581 | return "".join(r) | ||
|
r26857 | @wireprotocommand('clonebundles', '') | ||
def clonebundles(repo, proto): | ||||
"""Server command for returning info for available bundles to seed clones. | ||||
Clients will parse this response and determine what bundle to fetch. | ||||
Extensions may wrap this command to filter or dynamically emit data | ||||
depending on the request. e.g. you could advertise URLs for the closest | ||||
data center given the client's IP address. | ||||
""" | ||||
|
r31146 | return repo.vfs.tryread('clonebundles.manifest') | ||
|
r20774 | |||
wireprotocaps = ['lookup', 'changegroupsubset', 'branchmap', 'pushkey', | ||||
'known', 'getbundle', 'unbundlehash', 'batch'] | ||||
|
r20775 | |||
def _capabilities(repo, proto): | ||||
"""return a list of capabilities for a repo | ||||
This function exists to allow extensions to easily wrap capabilities | ||||
computation | ||||
- returns a lists: easy to alter | ||||
- change done here will be propagated to both `capabilities` and `hello` | ||||
|
r21024 | command without any other action needed. | ||
|
r20775 | """ | ||
|
r20774 | # copy to prevent modification of the global list | ||
caps = list(wireprotocaps) | ||||
|
r32744 | if streamclone.allowservergeneration(repo): | ||
r33222 | if repo.ui.configbool('server', 'preferuncompressed'): | |||
|
r16361 | caps.append('stream-preferred') | ||
|
r12296 | requiredformats = repo.requirements & repo.supportedformats | ||
# if our local revlogs are just revlogv1, add 'stream' cap | ||||
|
r32291 | if not requiredformats - {'revlogv1'}: | ||
|
r12296 | caps.append('stream') | ||
# otherwise, add 'streamreqs' detailing our local revlog format | ||||
else: | ||||
|
r26911 | caps.append('streamreqs=%s' % ','.join(sorted(requiredformats))) | ||
|
r33499 | if repo.ui.configbool('experimental', 'bundle2-advertise'): | ||
|
r22342 | capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo)) | ||
|
r28883 | caps.append('bundle2=' + urlreq.quote(capsblob)) | ||
|
r28666 | caps.append('unbundle=%s' % ','.join(bundle2.bundlepriority)) | ||
|
r30563 | |||
if proto.name == 'http': | ||||
caps.append('httpheader=%d' % | ||||
r33221 | repo.ui.configint('server', 'maxhttpheaderlen')) | |||
|
r33499 | if repo.ui.configbool('experimental', 'httppostargs'): | ||
|
r30563 | caps.append('httppostargs') | ||
|
r30762 | # FUTURE advertise 0.2rx once support is implemented | ||
# FUTURE advertise minrx and mintx after consulting config option | ||||
caps.append('httpmediatype=0.1rx,0.1tx,0.2tx') | ||||
compengines = supportedcompengines(repo.ui, proto, util.SERVERROLE) | ||||
if compengines: | ||||
comptypes = ','.join(urlreq.quote(e.wireprotosupport().name) | ||||
for e in compengines) | ||||
caps.append('compression=%s' % comptypes) | ||||
|
r20775 | return caps | ||
|
r21024 | # If you are writing an extension and consider wrapping this function. Wrap | ||
|
r20775 | # `_capabilities` instead. | ||
|
r20911 | @wireprotocommand('capabilities') | ||
|
r20775 | def capabilities(repo, proto): | ||
return ' '.join(_capabilities(repo, proto)) | ||||
|
r11594 | |||
|
r20912 | @wireprotocommand('changegroup', 'roots') | ||
|
r11584 | def changegroup(repo, proto, roots): | ||
|
r11597 | nodes = decodelist(roots) | ||
|
r34102 | outgoing = discovery.outgoing(repo, missingroots=nodes, | ||
missingheads=repo.heads()) | ||||
cg = changegroupmod.makechangegroup(repo, outgoing, '01', 'serve') | ||||
|
r30466 | return streamres(reader=cg, v1compressible=True) | ||
|
r11584 | |||
|
r20913 | @wireprotocommand('changegroupsubset', 'bases heads') | ||
|
r11584 | def changegroupsubset(repo, proto, bases, heads): | ||
|
r11597 | bases = decodelist(bases) | ||
heads = decodelist(heads) | ||||
|
r34099 | outgoing = discovery.outgoing(repo, missingroots=bases, | ||
missingheads=heads) | ||||
cg = changegroupmod.makechangegroup(repo, outgoing, '01', 'serve') | ||||
|
r30466 | return streamres(reader=cg, v1compressible=True) | ||
|
r11584 | |||
|
r20914 | @wireprotocommand('debugwireargs', 'one two *') | ||
|
r13721 | def debugwireargs(repo, proto, one, two, others): | ||
# only accept optional args from the known set | ||||
opts = options('debugwireargs', ['three', 'four'], others) | ||||
|
r35375 | return repo.debugwireargs(one, two, **pycompat.strkwargs(opts)) | ||
|
r13720 | |||
|
r20915 | @wireprotocommand('getbundle', '*') | ||
|
r13741 | def getbundle(repo, proto, others): | ||
|
r21646 | opts = options('getbundle', gboptsmap.keys(), others) | ||
|
r13741 | for k, v in opts.iteritems(): | ||
|
r21646 | keytype = gboptsmap[k] | ||
if keytype == 'nodes': | ||||
|
r19201 | opts[k] = decodelist(v) | ||
|
r21646 | elif keytype == 'csv': | ||
|
r25403 | opts[k] = list(v.split(',')) | ||
elif keytype == 'scsv': | ||||
|
r19201 | opts[k] = set(v.split(',')) | ||
|
r21988 | elif keytype == 'boolean': | ||
|
r26686 | # Client should serialize False as '0', which is a non-empty string | ||
# so it evaluates as a True bool. | ||||
if v == '0': | ||||
opts[k] = False | ||||
else: | ||||
opts[k] = bool(v) | ||||
|
r21646 | elif keytype != 'plain': | ||
raise KeyError('unknown getbundle option type %s' | ||||
% keytype) | ||||
|
r27246 | |||
|
r27633 | if not bundle1allowed(repo, 'pull'): | ||
|
r27246 | if not exchange.bundle2requested(opts.get('bundlecaps')): | ||
|
r30912 | if proto.name == 'http': | ||
return ooberror(bundle2required) | ||||
raise error.Abort(bundle2requiredmain, | ||||
hint=bundle2requiredhint) | ||||
|
r27246 | |||
|
r30914 | try: | ||
r33220 | if repo.ui.configbool('server', 'disablefullbundle'): | |||
|
r32260 | # Check to see if this is a full clone. | ||
clheads = set(repo.changelog.heads()) | ||||
heads = set(opts.get('heads', set())) | ||||
common = set(opts.get('common', set())) | ||||
common.discard(nullid) | ||||
if not common and clheads == heads: | ||||
raise error.Abort( | ||||
_('server has pull-based clones disabled'), | ||||
hint=_('remove --pull if specified or upgrade Mercurial')) | ||||
|
r34742 | chunks = exchange.getbundlechunks(repo, 'serve', | ||
**pycompat.strkwargs(opts)) | ||||
|
r30914 | except error.Abort as exc: | ||
# cleanly forward Abort error to the client | ||||
if not exchange.bundle2requested(opts.get('bundlecaps')): | ||||
if proto.name == 'http': | ||||
return ooberror(str(exc) + '\n') | ||||
raise # cannot do better for bundle1 + ssh | ||||
# bundle2 request expect a bundle2 reply | ||||
bundler = bundle2.bundle20(repo.ui) | ||||
manargs = [('message', str(exc))] | ||||
advargs = [] | ||||
if exc.hint is not None: | ||||
advargs.append(('hint', exc.hint)) | ||||
bundler.addpart(bundle2.bundlepart('error:abort', | ||||
manargs, advargs)) | ||||
return streamres(gen=bundler.getchunks(), v1compressible=True) | ||||
|
r30466 | return streamres(gen=chunks, v1compressible=True) | ||
|
r13741 | |||
|
r20916 | @wireprotocommand('heads') | ||
|
r11583 | def heads(repo, proto): | ||
|
r18281 | h = repo.heads() | ||
|
r11597 | return encodelist(h) + "\n" | ||
|
r11581 | |||
|
r20917 | @wireprotocommand('hello') | ||
|
r11594 | def hello(repo, proto): | ||
'''the hello command returns a set of lines describing various | ||||
interesting things about the server, in an RFC822-like format. | ||||
Currently the only one defined is "capabilities", which | ||||
consists of a line in the form: | ||||
capabilities: space separated list of tokens | ||||
''' | ||||
return "capabilities: %s\n" % (capabilities(repo, proto)) | ||||
|
r20919 | @wireprotocommand('listkeys', 'namespace') | ||
|
r11583 | def listkeys(repo, proto, namespace): | ||
|
r15217 | d = repo.listkeys(encoding.tolocal(namespace)).items() | ||
|
r21651 | return pushkeymod.encodekeys(d) | ||
|
r11581 | |||
|
r20920 | @wireprotocommand('lookup', 'key') | ||
|
r11583 | def lookup(repo, proto, key): | ||
|
r11581 | try: | ||
|
r15925 | k = encoding.tolocal(key) | ||
c = repo[k] | ||||
r = c.hex() | ||||
|
r11581 | success = 1 | ||
|
r25660 | except Exception as inst: | ||
|
r11581 | r = str(inst) | ||
success = 0 | ||||
|
r34732 | return "%d %s\n" % (success, r) | ||
|
r11581 | |||
|
r20918 | @wireprotocommand('known', 'nodes *') | ||
|
r14436 | def known(repo, proto, nodes, others): | ||
|
r13723 | return ''.join(b and "1" or "0" for b in repo.known(decodelist(nodes))) | ||
|
r20921 | @wireprotocommand('pushkey', 'namespace key old new') | ||
|
r11583 | def pushkey(repo, proto, namespace, key, old, new): | ||
|
r13050 | # compatibility with pre-1.8 clients which were accidentally | ||
# sending raw binary nodes rather than utf-8-encoded hex | ||||
|
r31451 | if len(new) == 20 and util.escapestr(new) != new: | ||
|
r13050 | # looks like it could be a binary node | ||
try: | ||||
|
r14064 | new.decode('utf-8') | ||
|
r13050 | new = encoding.tolocal(new) # but cleanly decodes as UTF-8 | ||
except UnicodeDecodeError: | ||||
pass # binary, leave unmodified | ||||
else: | ||||
new = encoding.tolocal(new) # normal path | ||||
|
r17793 | if util.safehasattr(proto, 'restore'): | ||
proto.redirect() | ||||
try: | ||||
r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key), | ||||
encoding.tolocal(old), new) or False | ||||
|
r26587 | except error.Abort: | ||
|
r17793 | r = False | ||
output = proto.restore() | ||||
return '%s\n%s' % (int(r), output) | ||||
|
r15217 | r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key), | ||
encoding.tolocal(old), new) | ||||
|
r11581 | return '%s\n' % int(r) | ||
|
r20922 | @wireprotocommand('stream_out') | ||
|
r11585 | def stream(repo, proto): | ||
|
r11627 | '''If the server supports streaming clone, it advertises the "stream" | ||
capability with a value representing the version and flags of the repo | ||||
it is serving. Client checks to see if it understands the format. | ||||
''' | ||||
|
r35507 | return streamres(streamclone.generatev1wireproto(repo)) | ||
|
r11585 | |||
|
r20923 | @wireprotocommand('unbundle', 'heads') | ||
|
r11593 | def unbundle(repo, proto, heads): | ||
|
r11597 | their_heads = decodelist(heads) | ||
|
r11593 | |||
|
r20967 | try: | ||
proto.redirect() | ||||
|
r11593 | |||
|
r20967 | exchange.check_heads(repo, their_heads, 'preparing changes') | ||
|
r12702 | |||
|
r20967 | # write bundle data to temporary file because it can be big | ||
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-') | ||||
|
r30924 | fp = os.fdopen(fd, pycompat.sysstr('wb+')) | ||
|
r20967 | r = 0 | ||
|
r11593 | try: | ||
|
r20967 | proto.getfile(fp) | ||
|
r20968 | fp.seek(0) | ||
|
r21064 | gen = exchange.readbundle(repo.ui, fp, None) | ||
|
r27246 | if (isinstance(gen, changegroupmod.cg1unpacker) | ||
|
r27633 | and not bundle1allowed(repo, 'push')): | ||
|
r30909 | if proto.name == 'http': | ||
# need to special case http because stderr do not get to | ||||
# the http client on failed push so we need to abuse some | ||||
# other error type to make sure the message get to the | ||||
# user. | ||||
return ooberror(bundle2required) | ||||
raise error.Abort(bundle2requiredmain, | ||||
hint=bundle2requiredhint) | ||||
|
r27246 | |||
|
r20968 | r = exchange.unbundle(repo, gen, their_heads, 'serve', | ||
proto._client()) | ||||
|
r21075 | if util.safehasattr(r, 'addpart'): | ||
|
r23139 | # The return looks streamable, we are in the bundle2 case and | ||
|
r21075 | # should return a stream. | ||
|
r30466 | return streamres(gen=r.getchunks()) | ||
|
r20967 | return pushres(r) | ||
|
r11593 | finally: | ||
|
r20967 | fp.close() | ||
os.unlink(tempname) | ||||
|
r24796 | |||
|
r26587 | except (error.BundleValueError, error.Abort, error.PushRaced) as exc: | ||
|
r24796 | # handle non-bundle2 case first | ||
if not getattr(exc, 'duringunbundle2', False): | ||||
try: | ||||
raise | ||||
|
r26587 | except error.Abort: | ||
|
r30473 | # The old code we moved used util.stderr directly. | ||
|
r24796 | # We did not change it to minimise code change. | ||
# This need to be moved to something proper. | ||||
# Feel free to do it. | ||||
|
r30473 | util.stderr.write("abort: %s\n" % exc) | ||
|
r30910 | if exc.hint is not None: | ||
util.stderr.write("(%s)\n" % exc.hint) | ||||
|
r24796 | return pushres(0) | ||
except error.PushRaced: | ||||
return pusherr(str(exc)) | ||||
bundler = bundle2.bundle20(repo.ui) | ||||
|
r24797 | for out in getattr(exc, '_bundle2salvagedoutput', ()): | ||
bundler.addpart(out) | ||||
|
r24796 | try: | ||
|
r25493 | try: | ||
raise | ||||
|
r25660 | except error.PushkeyFailed as exc: | ||
|
r25493 | # check client caps | ||
remotecaps = getattr(exc, '_replycaps', None) | ||||
if (remotecaps is not None | ||||
and 'pushkey' not in remotecaps.get('error', ())): | ||||
# no support remote side, fallback to Abort handler. | ||||
raise | ||||
part = bundler.newpart('error:pushkey') | ||||
part.addparam('in-reply-to', exc.partid) | ||||
if exc.namespace is not None: | ||||
part.addparam('namespace', exc.namespace, mandatory=False) | ||||
if exc.key is not None: | ||||
part.addparam('key', exc.key, mandatory=False) | ||||
if exc.new is not None: | ||||
part.addparam('new', exc.new, mandatory=False) | ||||
if exc.old is not None: | ||||
part.addparam('old', exc.old, mandatory=False) | ||||
if exc.ret is not None: | ||||
part.addparam('ret', exc.ret, mandatory=False) | ||||
|
r25660 | except error.BundleValueError as exc: | ||
|
r24686 | errpart = bundler.newpart('error:unsupportedcontent') | ||
|
r21627 | if exc.parttype is not None: | ||
errpart.addparam('parttype', exc.parttype) | ||||
|
r21622 | if exc.params: | ||
errpart.addparam('params', '\0'.join(exc.params)) | ||||
|
r26587 | except error.Abort as exc: | ||
|
r24796 | manargs = [('message', str(exc))] | ||
|
r21177 | advargs = [] | ||
|
r24796 | if exc.hint is not None: | ||
advargs.append(('hint', exc.hint)) | ||||
|
r24686 | bundler.addpart(bundle2.bundlepart('error:abort', | ||
|
r21177 | manargs, advargs)) | ||
|
r25660 | except error.PushRaced as exc: | ||
|
r24686 | bundler.newpart('error:pushraced', [('message', str(exc))]) | ||
|
r30466 | return streamres(gen=bundler.getchunks()) | ||