##// END OF EJS Templates
pack_dirstate: in C version, for invalidation set dict to what we write to disk...
pack_dirstate: in C version, for invalidation set dict to what we write to disk For files written out in the last second, Mercurial used to invalidate all the stat data (state, size, mode, mtime) while persisting to disk. This included invalidating the data in the dirstate dict as well. In commit 187bf2dde7c1, this was found to be unnecessary, and Mercurial switched to invalidating only the mtime. However, in the C version of pack_dirstate the value set in the dict was still the fully invalidated one. Switch to invalidating just the mtime in the dict as well.

File last commit:

r21728:0f73ed62 default
r21806:05bd2667 default
Show More
wireproto.py
863 lines | 28.8 KiB | text/x-python | PythonLexer
Matt Mackall
protocol: introduce wireproto.py...
r11581 # wireproto.py - generic wire protocol support functions
#
# Copyright 2005-2010 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Benoit Boissinot
fix undefined variables, spotted by pylint
r11879 import urllib, tempfile, os, sys
Matt Mackall
protocol: introduce wireproto.py...
r11581 from i18n import _
from node import bin, hex
Pierre-Yves David
wireproto: use pushkey.encodekey
r21651 import changegroup as changegroupmod, bundle2, pushkey as pushkeymod
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 import peer, error, encoding, util, store, exchange
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: introduce an abstractserverproto class...
r20903
class abstractserverproto(object):
"""abstract class that summarizes the protocol API
Used as reference and documentation.
"""
def getargs(self, args):
"""return the value for arguments in <args>
returns a list of values (same order as <args>)"""
raise NotImplementedError()
def getfile(self, fp):
"""write the whole content of a file into a file like object
The file is in the form::
(<chunk-size>\n<chunk>)+0\n
chunk size is the ascii version of the int.
"""
raise NotImplementedError()
def redirect(self):
"""may setup interception for stdout and stderr
See also the `restore` method."""
raise NotImplementedError()
# If the `redirect` function does install interception, the `restore`
# function MUST be defined. If interception is not used, this function
# MUST NOT be defined.
#
# left commented here on purpose
#
#def restore(self):
# """reinstall previous stdout and stderr and return intercepted stdout
# """
# raise NotImplementedError()
def groupchunks(self, cg):
"""return 4096 chunks from a changegroup object
Some protocols may have compressed the contents."""
raise NotImplementedError()
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 # abstract batching support
class future(object):
'''placeholder for a value to be set later'''
def set(self, value):
Augie Fackler
wireproto: use safehasattr or getattr instead of hasattr
r14970 if util.safehasattr(self, 'value'):
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 raise error.RepoError("future is already set")
self.value = value
class batcher(object):
'''base class for batches of commands submittable in a single request
Brodie Rao
cleanup: eradicate long lines
r16683 All methods invoked on instances of this class are simply queued and
return a a future for the result. Once you call submit(), all the queued
calls are performed and the results set in their respective futures.
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 '''
def __init__(self):
self.calls = []
def __getattr__(self, name):
def call(*args, **opts):
resref = future()
self.calls.append((name, args, opts, resref,))
return resref
return call
def submit(self):
pass
class localbatch(batcher):
'''performs the queued calls directly'''
def __init__(self, local):
batcher.__init__(self)
self.local = local
def submit(self):
for name, args, opts, resref in self.calls:
resref.set(getattr(self.local, name)(*args, **opts))
class remotebatch(batcher):
'''batches the queued calls; uses as few roundtrips as possible'''
def __init__(self, remote):
Brodie Rao
cleanup: eradicate long lines
r16683 '''remote must support _submitbatch(encbatch) and
_submitone(op, encargs)'''
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 batcher.__init__(self)
self.remote = remote
def submit(self):
req, rsp = [], []
for name, args, opts, resref in self.calls:
mtd = getattr(self.remote, name)
Augie Fackler
wireproto: use safehasattr or getattr instead of hasattr
r14970 batchablefn = getattr(mtd, 'batchable', None)
if batchablefn is not None:
batchable = batchablefn(mtd.im_self, *args, **opts)
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 encargsorres, encresref = batchable.next()
if encresref:
req.append((name, encargsorres,))
rsp.append((batchable, encresref, resref,))
else:
resref.set(encargsorres)
else:
if req:
self._submitreq(req, rsp)
req, rsp = [], []
resref.set(mtd(*args, **opts))
if req:
self._submitreq(req, rsp)
def _submitreq(self, req, rsp):
encresults = self.remote._submitbatch(req)
for encres, r in zip(encresults, rsp):
batchable, encresref, resref = r
encresref.set(encres)
resref.set(batchable.next())
def batchable(f):
'''annotation for batchable methods
Such methods must implement a coroutine as follows:
@batchable
def sample(self, one, two=None):
# Handle locally computable results first:
if not one:
yield "a local result", None
# Build list of encoded arguments suitable for your wire protocol:
encargs = [('one', encode(one),), ('two', encode(two),)]
# Create future for injection of encoded result:
encresref = future()
# Return encoded arguments and future:
yield encargs, encresref
Brodie Rao
cleanup: eradicate long lines
r16683 # Assuming the future to be filled with the result from the batched
# request now. Decode it:
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 yield decode(encresref.value)
Brodie Rao
cleanup: eradicate long lines
r16683 The decorator returns a function which wraps this coroutine as a plain
method, but adds the original method as an attribute called "batchable",
which is used by remotebatch to split the call into separate encoding and
decoding phases.
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 '''
def plain(*args, **opts):
batchable = f(*args, **opts)
encargsorres, encresref = batchable.next()
if not encresref:
return encargsorres # a local result in this case
self = args[0]
encresref.set(self._submitone(f.func_name, encargsorres))
return batchable.next()
setattr(plain, 'batchable', f)
return plain
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 # list of nodes encoding / decoding
def decodelist(l, sep=' '):
Peter Arrenbrecht
wireproto: fix decodelist to properly return empty list...
r13722 if l:
return map(bin, l.split(sep))
return []
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597
def encodelist(l, sep=' '):
return sep.join(map(hex, l))
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 # batched call argument encoding
def escapearg(plain):
return (plain
.replace(':', '::')
.replace(',', ':,')
.replace(';', ':;')
.replace('=', ':='))
def unescapearg(escaped):
return (escaped
.replace(':=', '=')
.replace(':;', ';')
.replace(':,', ',')
.replace('::', ':'))
Pierre-Yves David
getbundle: declare type of parameters...
r21646 # mapping of options accepted by getbundle and their types
#
# Meant to be extended by extensions. It is extensions responsibility to ensure
# such options are properly processed in exchange.getbundle.
#
# supported types are:
#
# :nodes: list of binary nodes
# :csv: list of comma-separated values
# :plain: string with no transformation needed.
gboptsmap = {'heads': 'nodes',
'common': 'nodes',
Pierre-Yves David
getbundle: support of listkeys argument when bundle2 is used...
r21657 'bundlecaps': 'csv',
'listkeys': 'csv'}
Pierre-Yves David
getbundle: declare type of parameters...
r21646
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 # client side
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 class wirepeer(peer.peerrepository):
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622
def batch(self):
return remotebatch(self)
def _submitbatch(self, req):
cmds = []
for op, argsdict in req:
args = ','.join('%s=%s' % p for p in argsdict.iteritems())
cmds.append('%s %s' % (op, args))
rsp = self._call("batch", cmds=';'.join(cmds))
return rsp.split(';')
def _submitone(self, op, args):
return self._call(op, **args)
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def lookup(self, key):
self.requirecap('lookup', _('look up remote revision'))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'key': encoding.fromlocal(key)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 success, data = d[:-1].split(" ", 1)
if int(success):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield bin(data)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 self._abort(error.RepoError(data))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def heads(self):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
yield {}, f
d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 try:
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield decodelist(d[:-1])
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Benoit Boissinot
fix undefined variables, spotted by pylint
r11879 self._abort(error.ResponseError(_("unexpected response:"), d))
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 def known(self, nodes):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'nodes': encodelist(nodes)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 try:
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield [bool(int(f)) for f in d]
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 self._abort(error.ResponseError(_("unexpected response:"), d))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def branchmap(self):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
yield {}, f
d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 try:
branchmap = {}
for branchpart in d.splitlines():
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchname, branchheads = branchpart.split(' ', 1)
Matt Mackall
branch: operate on branch names in local string space where possible...
r13047 branchname = encoding.tolocal(urllib.unquote(branchname))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchheads = decodelist(branchheads)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 branchmap[branchname] = branchheads
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield branchmap
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 except TypeError:
self._abort(error.ResponseError(_("unexpected response:"), d))
def branches(self, nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = encodelist(nodes)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 d = self._call("branches", nodes=n)
try:
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 br = [tuple(decodelist(b)) for b in d.splitlines()]
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 return br
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 self._abort(error.ResponseError(_("unexpected response:"), d))
def between(self, pairs):
Matt Mackall
protocol: unify basic http client requests
r11587 batch = 8 # avoid giant requests
r = []
for i in xrange(0, len(pairs), batch):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = " ".join([encodelist(p, '-') for p in pairs[i:i + batch]])
Matt Mackall
protocol: unify basic http client requests
r11587 d = self._call("between", pairs=n)
try:
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.extend(l and decodelist(l) or [] for l in d.splitlines())
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Matt Mackall
protocol: unify basic http client requests
r11587 self._abort(error.ResponseError(_("unexpected response:"), d))
return r
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def pushkey(self, namespace, key, old, new):
if not self.capable('pushkey'):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield False, None
f = future()
Pierre-Yves David
pushkey: add more verbose debug output regarding pushkey...
r17293 self.ui.debug('preparing pushkey for "%s:%s"\n' % (namespace, key))
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'namespace': encoding.fromlocal(namespace),
'key': encoding.fromlocal(key),
'old': encoding.fromlocal(old),
'new': encoding.fromlocal(new)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Pierre-Yves David
wireproto: handle other server output in pushkey...
r15652 d, output = d.split('\n', 1)
David Soria Parra
wireproto: catch possible cast error in pushkey...
r13450 try:
d = bool(int(d))
except ValueError:
raise error.ResponseError(
_('push failed (unexpected response):'), d)
Pierre-Yves David
wireproto: handle other server output in pushkey...
r15652 for l in output.splitlines(True):
self.ui.status(_('remote: '), l)
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield d
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def listkeys(self, namespace):
if not self.capable('pushkey'):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield {}, None
f = future()
Pierre-Yves David
pushkey: add more verbose debug output regarding pushkey...
r17293 self.ui.debug('preparing listkeys for "%s"\n' % namespace)
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'namespace': encoding.fromlocal(namespace)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Pierre-Yves David
wireproto: use pushkey.decodekey
r21653 yield pushkeymod.decodekeys(d)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Matt Mackall
protocol: unify stream_out client code
r11588 def stream_out(self):
return self._callstream('stream_out')
Matt Mackall
protocol: unify client changegroup methods
r11591 def changegroup(self, nodes, kind):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = encodelist(nodes)
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("changegroup", roots=n)
return changegroupmod.unbundle10(f, 'UN')
Matt Mackall
protocol: unify client changegroup methods
r11591
def changegroupsubset(self, bases, heads, kind):
self.requirecap('changegroupsubset', _('look up remote changes'))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 bases = encodelist(bases)
heads = encodelist(heads)
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("changegroupsubset",
bases=bases, heads=heads)
return changegroupmod.unbundle10(f, 'UN')
Matt Mackall
protocol: unify client changegroup methods
r11591
Pierre-Yves David
getbundle: declare type of parameters...
r21646 def getbundle(self, source, **kwargs):
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 self.requirecap('getbundle', _('look up remote changes'))
opts = {}
Pierre-Yves David
getbundle: declare type of parameters...
r21646 for key, value in kwargs.iteritems():
if value is None:
continue
keytype = gboptsmap.get(key)
if keytype is None:
assert False, 'unexpected'
elif keytype == 'nodes':
value = encodelist(value)
elif keytype == 'csv':
value = ','.join(value)
elif keytype != 'plain':
raise KeyError('unknown getbundle option type %s'
% keytype)
opts[key] = value
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("getbundle", **opts)
Pierre-Yves David
getbundle: declare type of parameters...
r21646 bundlecaps = kwargs.get('bundlecaps')
Pierre-Yves David
bundle2: use HG2X in the header...
r21144 if bundlecaps is not None and 'HG2X' in bundlecaps:
Pierre-Yves David
bundle2: allow bundle2 for pulling over the wire...
r21069 return bundle2.unbundle20(self.ui, f)
else:
return changegroupmod.unbundle10(f, 'UN')
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741
Matt Mackall
protocol: unify client unbundle support...
r11592 def unbundle(self, cg, heads, source):
'''Send cg (a readable file-like object representing the
changegroup to push, typically a chunkbuffer object) to the
Pierre-Yves David
bundle2: support for push over the wire...
r21075 remote server as a bundle.
When pushing a bundle10 stream, return an integer indicating the
result of the push (see localrepository.addchangegroup()).
When pushing a bundle20 stream, return a bundle20 stream.'''
Matt Mackall
protocol: unify client unbundle support...
r11592
Martin Geisler
wireproto: do not hash when heads == ['force']...
r14419 if heads != ['force'] and self.capable('unbundlehash'):
Shuhei Takahashi
wireproto: allow unbundle with hashed heads parameter (issue2126)...
r13942 heads = encodelist(['hashed',
util.sha1(''.join(sorted(heads))).digest()])
else:
heads = encodelist(heads)
Pierre-Yves David
bundle2: support for push over the wire...
r21075 if util.safehasattr(cg, 'deltaheader'):
# this a bundle10, do the old style call sequence
ret, output = self._callpush("unbundle", cg, heads=heads)
if ret == "":
raise error.ResponseError(
_('push failed:'), output)
try:
ret = int(ret)
except ValueError:
raise error.ResponseError(
_('push failed (unexpected response):'), ret)
Matt Mackall
protocol: unify client unbundle support...
r11592
Pierre-Yves David
bundle2: support for push over the wire...
r21075 for l in output.splitlines(True):
self.ui.status(_('remote: '), l)
else:
# bundle2 push. Send a stream, fetch a stream.
stream = self._calltwowaystream('unbundle', cg, heads=heads)
ret = bundle2.unbundle20(self.ui, stream)
Matt Mackall
protocol: unify client unbundle support...
r11592 return ret
Peter Arrenbrecht
wireproto: add test for new optional arg missing on server...
r14048 def debugwireargs(self, one, two, three=None, four=None, five=None):
Peter Arrenbrecht
debug: add debugwireargs to test argument passing over the wire...
r13720 # don't pass optional arguments left at their default value
opts = {}
if three is not None:
opts['three'] = three
if four is not None:
opts['four'] = four
return self._call('debugwireargs', one=one, two=two, **opts)
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _call(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a simple string.
returns the server reply as a string."""
raise NotImplementedError()
def _callstream(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a stream.
returns the server reply as a file like object."""
raise NotImplementedError()
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 def _callcompressable(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a stream.
Mads Kiilerich
spelling: fixes from spell checker
r21024 The stream may have been compressed in some implementations. This
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 function takes care of the decompression. This is the only difference
with _callstream.
returns the server reply as a file like object.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _callpush(self, cmd, fp, **args):
"""execute a <cmd> on server
The command is expected to be related to a push. Push has a special
return method.
returns the server reply as a (ret, output) tuple. ret is either
empty (error) or a stringified int.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: add a _calltwowaystream method on wirepeer...
r21072 def _calltwowaystream(self, cmd, fp, **args):
"""execute <cmd> on server
The command will send a stream to the server and get a stream in reply.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _abort(self, exception):
"""clearly abort the wire protocol connection and raise the exception
"""
raise NotImplementedError()
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 # server side
Pierre-Yves David
wireproto: document possible return type...
r20902 # wire protocol command can either return a string or one of these classes.
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 class streamres(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: binary stream
The call was successful and the result is a stream.
Iterate on the `self.gen` attribute to retrieve chunks.
"""
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 def __init__(self, gen):
self.gen = gen
class pushres(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: success with simple integer return
The call was successful and returned an integer contained in `self.res`.
"""
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 def __init__(self, res):
self.res = res
Benoit Boissinot
wireproto: introduce pusherr() to deal with "unsynced changes" error...
r12703 class pusherr(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: failure
The call failed. The `self.res` attribute contains the error message.
"""
Benoit Boissinot
wireproto: introduce pusherr() to deal with "unsynced changes" error...
r12703 def __init__(self, res):
self.res = res
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 class ooberror(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: failure of a batch of operation
Something failed during a batch call. The error message is stored in
`self.message`.
"""
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 def __init__(self, message):
self.message = message
Matt Mackall
protocol: introduce wireproto.py...
r11581 def dispatch(repo, proto, command):
Kevin Bullock
filtering: rename filters to their antonyms...
r18382 repo = repo.filtered("served")
Matt Mackall
protocol: introduce wireproto.py...
r11581 func, spec = commands[command]
args = proto.getargs(spec)
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return func(repo, proto, *args)
Matt Mackall
protocol: introduce wireproto.py...
r11581
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 def options(cmd, keys, others):
opts = {}
for k in keys:
if k in others:
opts[k] = others[k]
del others[k]
if others:
Pierre-Yves David
wireproto: rephrase the error message for unknown getbundle parameters...
r21728 sys.stderr.write("warning: %s ignored unexpected arguments %s\n"
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 % (cmd, ",".join(others)))
return opts
Pierre-Yves David
wireproto: add decorator for wire protocol command...
r20906 # list of commands
commands = {}
def wireprotocommand(name, args=''):
Mads Kiilerich
spelling: fixes from spell checker
r21024 """decorator for wire protocol command"""
Pierre-Yves David
wireproto: add decorator for wire protocol command...
r20906 def register(func):
commands[name] = (func, args)
return func
return register
Pierre-Yves David
wireproto: use decorator for the batch command
r20907 @wireprotocommand('batch', 'cmds *')
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 def batch(repo, proto, cmds, others):
Kevin Bullock
filtering: rename filters to their antonyms...
r18382 repo = repo.filtered("served")
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 res = []
for pair in cmds.split(';'):
op, args = pair.split(' ', 1)
vals = {}
for a in args.split(','):
if a:
n, v = a.split('=')
vals[n] = unescapearg(v)
func, spec = commands[op]
if spec:
keys = spec.split()
data = {}
for k in keys:
if k == '*':
star = {}
for key in vals.keys():
if key not in keys:
star[key] = vals[key]
data['*'] = star
else:
data[k] = vals[k]
result = func(repo, proto, *[data[k] for k in keys])
else:
result = func(repo, proto)
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 if isinstance(result, ooberror):
return result
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 res.append(escapearg(result))
return ';'.join(res)
Pierre-Yves David
wireproto: use decorator for the between command
r20908 @wireprotocommand('between', 'pairs')
Matt Mackall
protocol: add proto to method prototypes
r11583 def between(repo, proto, pairs):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 pairs = [decodelist(p, '-') for p in pairs.split(" ")]
Matt Mackall
protocol: introduce wireproto.py...
r11581 r = []
for b in repo.between(pairs):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.append(encodelist(b) + "\n")
Matt Mackall
protocol: introduce wireproto.py...
r11581 return "".join(r)
Pierre-Yves David
wireproto: use decorator for the branchmap command
r20909 @wireprotocommand('branchmap')
Matt Mackall
protocol: add proto to method prototypes
r11583 def branchmap(repo, proto):
Pierre-Yves David
clfilter: drop extra filtering in wireprotocol...
r18281 branchmap = repo.branchmap()
Matt Mackall
protocol: introduce wireproto.py...
r11581 heads = []
for branch, nodes in branchmap.iteritems():
Matt Mackall
branch: operate on branch names in local string space where possible...
r13047 branchname = urllib.quote(encoding.fromlocal(branch))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchnodes = encodelist(nodes)
heads.append('%s %s' % (branchname, branchnodes))
Matt Mackall
protocol: introduce wireproto.py...
r11581 return '\n'.join(heads)
Pierre-Yves David
wireproto: use decorator for the branches command
r20910 @wireprotocommand('branches', 'nodes')
Matt Mackall
protocol: add proto to method prototypes
r11583 def branches(repo, proto, nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 nodes = decodelist(nodes)
Matt Mackall
protocol: introduce wireproto.py...
r11581 r = []
for b in repo.branches(nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.append(encodelist(b) + "\n")
Matt Mackall
protocol: introduce wireproto.py...
r11581 return "".join(r)
Pierre-Yves David
wireproto: extract capabilities list in outside the wireproto function...
r20774
wireprotocaps = ['lookup', 'changegroupsubset', 'branchmap', 'pushkey',
'known', 'getbundle', 'unbundlehash', 'batch']
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775
def _capabilities(repo, proto):
"""return a list of capabilities for a repo
This function exists to allow extensions to easily wrap capabilities
computation
- returns a lists: easy to alter
- change done here will be propagated to both `capabilities` and `hello`
Mads Kiilerich
spelling: fixes from spell checker
r21024 command without any other action needed.
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 """
Pierre-Yves David
wireproto: extract capabilities list in outside the wireproto function...
r20774 # copy to prevent modification of the global list
caps = list(wireprotocaps)
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 if _allowstream(repo.ui):
Benoit Allard
protocol: Add the stream-preferred capability...
r16361 if repo.ui.configbool('server', 'preferuncompressed', False):
caps.append('stream-preferred')
Sune Foldager
clone: only use stream when we understand the revlog format...
r12296 requiredformats = repo.requirements & repo.supportedformats
# if our local revlogs are just revlogv1, add 'stream' cap
if not requiredformats - set(('revlogv1',)):
caps.append('stream')
# otherwise, add 'streamreqs' detailing our local revlog format
else:
caps.append('streamreqs=%s' % ','.join(requiredformats))
Pierre-Yves David
bundle2: require both client and server to opt in...
r21148 if repo.ui.configbool('experimental', 'bundle2-exp', False):
Pierre-Yves David
bundle2: advertise bundle2 caps in server capabilities...
r21141 capsblob = bundle2.encodecaps(repo.bundle2caps)
Pierre-Yves David
bundle2: rename server capability to bundle2-exp...
r21145 caps.append('bundle2-exp=' + urllib.quote(capsblob))
Matt Mackall
protocol: unify server-side capabilities functions
r11594 caps.append('unbundle=%s' % ','.join(changegroupmod.bundlepriority))
Steven Brown
httprepo: long arguments support (issue2126)...
r14093 caps.append('httpheader=1024')
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 return caps
Mads Kiilerich
spelling: fixes from spell checker
r21024 # If you are writing an extension and consider wrapping this function. Wrap
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 # `_capabilities` instead.
Pierre-Yves David
wireproto: use decorator for the capabilities command
r20911 @wireprotocommand('capabilities')
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 def capabilities(repo, proto):
return ' '.join(_capabilities(repo, proto))
Matt Mackall
protocol: unify server-side capabilities functions
r11594
Pierre-Yves David
wireproto: use decorator for the changegroup command
r20912 @wireprotocommand('changegroup', 'roots')
Matt Mackall
protocol: unify changegroup commands...
r11584 def changegroup(repo, proto, roots):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 nodes = decodelist(roots)
Pierre-Yves David
localrepo: move the changegroup method in changegroup module...
r20931 cg = changegroupmod.changegroup(repo, nodes, 'serve')
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return streamres(proto.groupchunks(cg))
Matt Mackall
protocol: unify changegroup commands...
r11584
Pierre-Yves David
wireproto: use decorator for the changegroupsubset command
r20913 @wireprotocommand('changegroupsubset', 'bases heads')
Matt Mackall
protocol: unify changegroup commands...
r11584 def changegroupsubset(repo, proto, bases, heads):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 bases = decodelist(bases)
heads = decodelist(heads)
Pierre-Yves David
localrepo: move the changegroupsubset method in changegroup module...
r20927 cg = changegroupmod.changegroupsubset(repo, bases, heads, 'serve')
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return streamres(proto.groupchunks(cg))
Matt Mackall
protocol: unify changegroup commands...
r11584
Pierre-Yves David
wireproto: use decorator for the debugwireargs command
r20914 @wireprotocommand('debugwireargs', 'one two *')
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 def debugwireargs(repo, proto, one, two, others):
# only accept optional args from the known set
opts = options('debugwireargs', ['three', 'four'], others)
return repo.debugwireargs(one, two, **opts)
Peter Arrenbrecht
debug: add debugwireargs to test argument passing over the wire...
r13720
Pierre-Yves David
wireproto: expose the list of getbundle arguments to extensions...
r21615 # List of options accepted by getbundle.
#
# Meant to be extended by extensions. It is the extension's responsibility to
# ensure such options are properly processed in exchange.getbundle.
gboptslist = ['heads', 'common', 'bundlecaps']
Pierre-Yves David
wireproto: use decorator for the getbundle command
r20915 @wireprotocommand('getbundle', '*')
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 def getbundle(repo, proto, others):
Pierre-Yves David
getbundle: declare type of parameters...
r21646 opts = options('getbundle', gboptsmap.keys(), others)
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 for k, v in opts.iteritems():
Pierre-Yves David
getbundle: declare type of parameters...
r21646 keytype = gboptsmap[k]
if keytype == 'nodes':
Benoit Boissinot
bundle-ng: add bundlecaps argument to getbundle() command
r19201 opts[k] = decodelist(v)
Pierre-Yves David
getbundle: declare type of parameters...
r21646 elif keytype == 'csv':
Benoit Boissinot
bundle-ng: add bundlecaps argument to getbundle() command
r19201 opts[k] = set(v.split(','))
Pierre-Yves David
getbundle: declare type of parameters...
r21646 elif keytype != 'plain':
raise KeyError('unknown getbundle option type %s'
% keytype)
Pierre-Yves David
bundle2: allow bundle2 for pulling over the wire...
r21069 cg = exchange.getbundle(repo, 'serve', **opts)
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 return streamres(proto.groupchunks(cg))
Pierre-Yves David
wireproto: use decorator for the heads command
r20916 @wireprotocommand('heads')
Matt Mackall
protocol: add proto to method prototypes
r11583 def heads(repo, proto):
Pierre-Yves David
clfilter: drop extra filtering in wireprotocol...
r18281 h = repo.heads()
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 return encodelist(h) + "\n"
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: use decorator for the hello command
r20917 @wireprotocommand('hello')
Matt Mackall
protocol: unify server-side capabilities functions
r11594 def hello(repo, proto):
'''the hello command returns a set of lines describing various
interesting things about the server, in an RFC822-like format.
Currently the only one defined is "capabilities", which
consists of a line in the form:
capabilities: space separated list of tokens
'''
return "capabilities: %s\n" % (capabilities(repo, proto))
Pierre-Yves David
wireproto: use decorator for the listkeys command
r20919 @wireprotocommand('listkeys', 'namespace')
Matt Mackall
protocol: add proto to method prototypes
r11583 def listkeys(repo, proto, namespace):
Andreas Freimuth
wireproto: do not call pushkey module directly (issue3041)...
r15217 d = repo.listkeys(encoding.tolocal(namespace)).items()
Pierre-Yves David
wireproto: use pushkey.encodekey
r21651 return pushkeymod.encodekeys(d)
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: use decorator for the lookup command
r20920 @wireprotocommand('lookup', 'key')
Matt Mackall
protocol: add proto to method prototypes
r11583 def lookup(repo, proto, key):
Matt Mackall
protocol: introduce wireproto.py...
r11581 try:
Matt Mackall
wireproto: refuse to lookup secret csets
r15925 k = encoding.tolocal(key)
c = repo[k]
r = c.hex()
Matt Mackall
protocol: introduce wireproto.py...
r11581 success = 1
except Exception, inst:
r = str(inst)
success = 0
return "%s %s\n" % (success, r)
Pierre-Yves David
wireproto: use decorator for the known command
r20918 @wireprotocommand('known', 'nodes *')
Peter Arrenbrecht
wireproto: enable optional args for known() for future extensibility...
r14436 def known(repo, proto, nodes, others):
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 return ''.join(b and "1" or "0" for b in repo.known(decodelist(nodes)))
Pierre-Yves David
wireproto: use decorator for the pushkey command
r20921 @wireprotocommand('pushkey', 'namespace key old new')
Matt Mackall
protocol: add proto to method prototypes
r11583 def pushkey(repo, proto, namespace, key, old, new):
Matt Mackall
pushkey: use UTF-8
r13050 # compatibility with pre-1.8 clients which were accidentally
# sending raw binary nodes rather than utf-8-encoded hex
if len(new) == 20 and new.encode('string-escape') != new:
# looks like it could be a binary node
try:
Alexander Solovyov
remove unused imports and variables
r14064 new.decode('utf-8')
Matt Mackall
pushkey: use UTF-8
r13050 new = encoding.tolocal(new) # but cleanly decodes as UTF-8
except UnicodeDecodeError:
pass # binary, leave unmodified
else:
new = encoding.tolocal(new) # normal path
Wagner Bruna
wireproto: fix pushkey hook failure and output on remote http repo...
r17793 if util.safehasattr(proto, 'restore'):
proto.redirect()
try:
r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key),
encoding.tolocal(old), new) or False
except util.Abort:
r = False
output = proto.restore()
return '%s\n%s' % (int(r), output)
Andreas Freimuth
wireproto: do not call pushkey module directly (issue3041)...
r15217 r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key),
encoding.tolocal(old), new)
Matt Mackall
protocol: introduce wireproto.py...
r11581 return '%s\n' % int(r)
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 def _allowstream(ui):
return ui.configbool('server', 'uncompressed', True, untrusted=True)
Durham Goode
clone: move file stream walk to a separate function...
r19176 def _walkstreamfiles(repo):
# this is it's own function so extensions can override it
return repo.store.walk()
Pierre-Yves David
wireproto: use decorator for the stream command
r20922 @wireprotocommand('stream_out')
Matt Mackall
protocol: unify stream_out command
r11585 def stream(repo, proto):
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 '''If the server supports streaming clone, it advertises the "stream"
capability with a value representing the version and flags of the repo
it is serving. Client checks to see if it understands the format.
The format is simple: the server writes out a line with the amount
Mads Kiilerich
fix trivial spelling errors
r17424 of files, then the total amount of bytes to be transferred (separated
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 by a space). Then, for each file, the server first writes the filename
Mads Kiilerich
spelling: fixes from spell checker
r21024 and file size (separated by the null character), then the file contents.
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 '''
if not _allowstream(repo.ui):
return '1\n'
entries = []
total_bytes = 0
try:
# get consistent snapshot of repo, lock during scan
lock = repo.lock()
try:
repo.ui.debug('scanning\n')
Durham Goode
clone: move file stream walk to a separate function...
r19176 for name, ename, size in _walkstreamfiles(repo):
Mads Kiilerich
clone: don't include empty revlogs in stream
r18381 if size:
entries.append((name, size))
total_bytes += size
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 finally:
lock.release()
except error.LockError:
return '2\n' # error: 2
def streamer(repo, entries, total):
'''stream out all metadata files in repository.'''
yield '0\n' # success
repo.ui.debug('%d files, %d bytes to transfer\n' %
(len(entries), total_bytes))
yield '%d %d\n' % (len(entries), total_bytes)
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556
sopener = repo.sopener
oldaudit = sopener.mustaudit
Bryan O'Sullivan
wireproto: don't format a debug string inside a hot loop...
r17558 debugflag = repo.ui.debugflag
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 sopener.mustaudit = False
try:
for name, size in entries:
Bryan O'Sullivan
wireproto: don't format a debug string inside a hot loop...
r17558 if debugflag:
repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 # partially encode name over the wire for backwards compat
yield '%s\0%d\n' % (store.encodedir(name), size)
Bryan O'Sullivan
wireproto: bypass filechunkiter for small files when streaming...
r17557 if size <= 65536:
Patrick Mezard
wireproto: fix check-code.py breakage introduced by 6d97dd630d79
r17567 fp = sopener(name)
try:
data = fp.read(size)
finally:
fp.close()
yield data
Bryan O'Sullivan
wireproto: bypass filechunkiter for small files when streaming...
r17557 else:
for chunk in util.filechunkiter(sopener(name), limit=size):
yield chunk
Thomas Arendsen Hein
wireproto: workaround for yield inside try/finally incompatible with python2.4
r17603 # replace with "finally:" when support for python 2.4 has been dropped
except Exception:
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 sopener.mustaudit = oldaudit
Thomas Arendsen Hein
wireproto: workaround for yield inside try/finally incompatible with python2.4
r17603 raise
sopener.mustaudit = oldaudit
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627
return streamres(streamer(repo, entries, total_bytes))
Matt Mackall
protocol: unify stream_out command
r11585
Pierre-Yves David
wireproto: use decorator for the ubundle command
r20923 @wireprotocommand('unbundle', 'heads')
Matt Mackall
protocol: unify unbundle on the server side
r11593 def unbundle(repo, proto, heads):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 their_heads = decodelist(heads)
Matt Mackall
protocol: unify unbundle on the server side
r11593
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 try:
proto.redirect()
Matt Mackall
protocol: unify unbundle on the server side
r11593
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 exchange.check_heads(repo, their_heads, 'preparing changes')
Benoit Boissinot
wireproto: redirect the output earlier
r12702
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 # write bundle data to temporary file because it can be big
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
fp = os.fdopen(fd, 'wb+')
r = 0
Matt Mackall
protocol: unify unbundle on the server side
r11593 try:
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 proto.getfile(fp)
Pierre-Yves David
unbundle: extract the core logic in another function...
r20968 fp.seek(0)
Pierre-Yves David
bundle2: add a ui argument to readbundle...
r21064 gen = exchange.readbundle(repo.ui, fp, None)
Pierre-Yves David
unbundle: extract the core logic in another function...
r20968 r = exchange.unbundle(repo, gen, their_heads, 'serve',
proto._client())
Pierre-Yves David
bundle2: support for push over the wire...
r21075 if util.safehasattr(r, 'addpart'):
# The return looks streameable, we are in the bundle2 case and
# should return a stream.
return streamres(r.getchunks())
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 return pushres(r)
Matt Mackall
protocol: unify unbundle on the server side
r11593 finally:
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 fp.close()
os.unlink(tempname)
Pierre-Yves David
bundle2: move exception classes into the error module...
r21618 except error.BundleValueError, exc:
Pierre-Yves David
bundle2: gracefully handle UnknownPartError during unbundle...
r21183 bundler = bundle2.bundle20(repo.ui)
Pierre-Yves David
bundle2: rename b2x:error:unknownpart to b2x:error:unsupportedcontent...
r21619 errpart = bundler.newpart('B2X:ERROR:UNSUPPORTEDCONTENT')
Pierre-Yves David
bundle2: support None parttype in BundleValueError...
r21627 if exc.parttype is not None:
errpart.addparam('parttype', exc.parttype)
Pierre-Yves David
bundle2: support transmission of params error over the wire...
r21622 if exc.params:
errpart.addparam('params', '\0'.join(exc.params))
Pierre-Yves David
bundle2: gracefully handle UnknownPartError during unbundle...
r21183 return streamres(bundler.getchunks())
Pierre-Yves David
localrepo: add unbundle support...
r20969 except util.Abort, inst:
# The old code we moved used sys.stderr directly.
Mads Kiilerich
spelling: fixes from spell checker
r21024 # We did not change it to minimise code change.
Pierre-Yves David
localrepo: add unbundle support...
r20969 # This need to be moved to something proper.
# Feel free to do it.
Pierre-Yves David
bundle2: gracefully handle abort during unbundle...
r21177 if getattr(inst, 'duringunbundle2', False):
bundler = bundle2.bundle20(repo.ui)
manargs = [('message', str(inst))]
advargs = []
if inst.hint is not None:
advargs.append(('hint', inst.hint))
bundler.addpart(bundle2.bundlepart('B2X:ERROR:ABORT',
manargs, advargs))
return streamres(bundler.getchunks())
else:
sys.stderr.write("abort: %s\n" % inst)
return pushres(0)
Pierre-Yves David
bundle2: fix raising errors during heads checking...
r21184 except error.PushRaced, exc:
Pierre-Yves David
bundle2: gracefully handle PushRaced error during unbundle...
r21186 if getattr(exc, 'duringunbundle2', False):
bundler = bundle2.bundle20(repo.ui)
Pierre-Yves David
bundle2: update all ``addpart`` callers to ``newpart``...
r21600 bundler.newpart('B2X:ERROR:PUSHRACED', [('message', str(exc))])
Pierre-Yves David
bundle2: gracefully handle PushRaced error during unbundle...
r21186 return streamres(bundler.getchunks())
else:
return pusherr(str(exc))