##// END OF EJS Templates
parsers: inline fields of dirstate values in C version...
parsers: inline fields of dirstate values in C version Previously, while unpacking the dirstate we'd create 3-4 new CPython objects for most dirstate values: - the state is a single character string, which is pooled by CPython - the mode is a new object if it isn't 0 due to being in the lookup set - the size is a new object if it is greater than 255 - the mtime is a new object if it isn't -1 due to being in the lookup set - the tuple to contain them all In some cases such as regular hg status, we actually look at all the objects. In other cases like hg add, hg status for a subdirectory, or hg status with the third-party hgwatchman enabled, we look at almost none of the objects. This patch eliminates most object creation in these cases by defining a custom C struct that is exposed to Python with an interface similar to a tuple. Only when tuple elements are actually requested are the respective objects created. The gains, where they're expected, are significant. The following tests are run against a working copy with over 270,000 files. parse_dirstate becomes significantly faster: $ hg perfdirstate before: wall 0.186437 comb 0.180000 user 0.160000 sys 0.020000 (best of 35) after: wall 0.093158 comb 0.100000 user 0.090000 sys 0.010000 (best of 95) and as a result, several commands benefit: $ time hg status # with hgwatchman enabled before: 0.42s user 0.14s system 99% cpu 0.563 total after: 0.34s user 0.12s system 99% cpu 0.471 total $ time hg add new-file before: 0.85s user 0.18s system 99% cpu 1.033 total after: 0.76s user 0.17s system 99% cpu 0.931 total There is a slight regression in regular status performance, but this is fixed in an upcoming patch.

File last commit:

r21728:0f73ed62 default
r21809:e250b830 default
Show More
wireproto.py
863 lines | 28.8 KiB | text/x-python | PythonLexer
Matt Mackall
protocol: introduce wireproto.py...
r11581 # wireproto.py - generic wire protocol support functions
#
# Copyright 2005-2010 Matt Mackall <mpm@selenic.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Benoit Boissinot
fix undefined variables, spotted by pylint
r11879 import urllib, tempfile, os, sys
Matt Mackall
protocol: introduce wireproto.py...
r11581 from i18n import _
from node import bin, hex
Pierre-Yves David
wireproto: use pushkey.encodekey
r21651 import changegroup as changegroupmod, bundle2, pushkey as pushkeymod
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 import peer, error, encoding, util, store, exchange
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: introduce an abstractserverproto class...
r20903
class abstractserverproto(object):
"""abstract class that summarizes the protocol API
Used as reference and documentation.
"""
def getargs(self, args):
"""return the value for arguments in <args>
returns a list of values (same order as <args>)"""
raise NotImplementedError()
def getfile(self, fp):
"""write the whole content of a file into a file like object
The file is in the form::
(<chunk-size>\n<chunk>)+0\n
chunk size is the ascii version of the int.
"""
raise NotImplementedError()
def redirect(self):
"""may setup interception for stdout and stderr
See also the `restore` method."""
raise NotImplementedError()
# If the `redirect` function does install interception, the `restore`
# function MUST be defined. If interception is not used, this function
# MUST NOT be defined.
#
# left commented here on purpose
#
#def restore(self):
# """reinstall previous stdout and stderr and return intercepted stdout
# """
# raise NotImplementedError()
def groupchunks(self, cg):
"""return 4096 chunks from a changegroup object
Some protocols may have compressed the contents."""
raise NotImplementedError()
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 # abstract batching support
class future(object):
'''placeholder for a value to be set later'''
def set(self, value):
Augie Fackler
wireproto: use safehasattr or getattr instead of hasattr
r14970 if util.safehasattr(self, 'value'):
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 raise error.RepoError("future is already set")
self.value = value
class batcher(object):
'''base class for batches of commands submittable in a single request
Brodie Rao
cleanup: eradicate long lines
r16683 All methods invoked on instances of this class are simply queued and
return a a future for the result. Once you call submit(), all the queued
calls are performed and the results set in their respective futures.
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 '''
def __init__(self):
self.calls = []
def __getattr__(self, name):
def call(*args, **opts):
resref = future()
self.calls.append((name, args, opts, resref,))
return resref
return call
def submit(self):
pass
class localbatch(batcher):
'''performs the queued calls directly'''
def __init__(self, local):
batcher.__init__(self)
self.local = local
def submit(self):
for name, args, opts, resref in self.calls:
resref.set(getattr(self.local, name)(*args, **opts))
class remotebatch(batcher):
'''batches the queued calls; uses as few roundtrips as possible'''
def __init__(self, remote):
Brodie Rao
cleanup: eradicate long lines
r16683 '''remote must support _submitbatch(encbatch) and
_submitone(op, encargs)'''
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 batcher.__init__(self)
self.remote = remote
def submit(self):
req, rsp = [], []
for name, args, opts, resref in self.calls:
mtd = getattr(self.remote, name)
Augie Fackler
wireproto: use safehasattr or getattr instead of hasattr
r14970 batchablefn = getattr(mtd, 'batchable', None)
if batchablefn is not None:
batchable = batchablefn(mtd.im_self, *args, **opts)
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 encargsorres, encresref = batchable.next()
if encresref:
req.append((name, encargsorres,))
rsp.append((batchable, encresref, resref,))
else:
resref.set(encargsorres)
else:
if req:
self._submitreq(req, rsp)
req, rsp = [], []
resref.set(mtd(*args, **opts))
if req:
self._submitreq(req, rsp)
def _submitreq(self, req, rsp):
encresults = self.remote._submitbatch(req)
for encres, r in zip(encresults, rsp):
batchable, encresref, resref = r
encresref.set(encres)
resref.set(batchable.next())
def batchable(f):
'''annotation for batchable methods
Such methods must implement a coroutine as follows:
@batchable
def sample(self, one, two=None):
# Handle locally computable results first:
if not one:
yield "a local result", None
# Build list of encoded arguments suitable for your wire protocol:
encargs = [('one', encode(one),), ('two', encode(two),)]
# Create future for injection of encoded result:
encresref = future()
# Return encoded arguments and future:
yield encargs, encresref
Brodie Rao
cleanup: eradicate long lines
r16683 # Assuming the future to be filled with the result from the batched
# request now. Decode it:
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 yield decode(encresref.value)
Brodie Rao
cleanup: eradicate long lines
r16683 The decorator returns a function which wraps this coroutine as a plain
method, but adds the original method as an attribute called "batchable",
which is used by remotebatch to split the call into separate encoding and
decoding phases.
Peter Arrenbrecht
wireproto: add basic command batching infrastructure...
r14621 '''
def plain(*args, **opts):
batchable = f(*args, **opts)
encargsorres, encresref = batchable.next()
if not encresref:
return encargsorres # a local result in this case
self = args[0]
encresref.set(self._submitone(f.func_name, encargsorres))
return batchable.next()
setattr(plain, 'batchable', f)
return plain
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 # list of nodes encoding / decoding
def decodelist(l, sep=' '):
Peter Arrenbrecht
wireproto: fix decodelist to properly return empty list...
r13722 if l:
return map(bin, l.split(sep))
return []
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597
def encodelist(l, sep=' '):
return sep.join(map(hex, l))
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 # batched call argument encoding
def escapearg(plain):
return (plain
.replace(':', '::')
.replace(',', ':,')
.replace(';', ':;')
.replace('=', ':='))
def unescapearg(escaped):
return (escaped
.replace(':=', '=')
.replace(':;', ';')
.replace(':,', ',')
.replace('::', ':'))
Pierre-Yves David
getbundle: declare type of parameters...
r21646 # mapping of options accepted by getbundle and their types
#
# Meant to be extended by extensions. It is extensions responsibility to ensure
# such options are properly processed in exchange.getbundle.
#
# supported types are:
#
# :nodes: list of binary nodes
# :csv: list of comma-separated values
# :plain: string with no transformation needed.
gboptsmap = {'heads': 'nodes',
'common': 'nodes',
Pierre-Yves David
getbundle: support of listkeys argument when bundle2 is used...
r21657 'bundlecaps': 'csv',
'listkeys': 'csv'}
Pierre-Yves David
getbundle: declare type of parameters...
r21646
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 # client side
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 class wirepeer(peer.peerrepository):
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622
def batch(self):
return remotebatch(self)
def _submitbatch(self, req):
cmds = []
for op, argsdict in req:
args = ','.join('%s=%s' % p for p in argsdict.iteritems())
cmds.append('%s %s' % (op, args))
rsp = self._call("batch", cmds=';'.join(cmds))
return rsp.split(';')
def _submitone(self, op, args):
return self._call(op, **args)
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def lookup(self, key):
self.requirecap('lookup', _('look up remote revision'))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'key': encoding.fromlocal(key)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 success, data = d[:-1].split(" ", 1)
if int(success):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield bin(data)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 self._abort(error.RepoError(data))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def heads(self):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
yield {}, f
d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 try:
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield decodelist(d[:-1])
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Benoit Boissinot
fix undefined variables, spotted by pylint
r11879 self._abort(error.ResponseError(_("unexpected response:"), d))
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 def known(self, nodes):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'nodes': encodelist(nodes)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 try:
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield [bool(int(f)) for f in d]
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 self._abort(error.ResponseError(_("unexpected response:"), d))
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def branchmap(self):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 f = future()
yield {}, f
d = f.value
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 try:
branchmap = {}
for branchpart in d.splitlines():
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchname, branchheads = branchpart.split(' ', 1)
Matt Mackall
branch: operate on branch names in local string space where possible...
r13047 branchname = encoding.tolocal(urllib.unquote(branchname))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchheads = decodelist(branchheads)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 branchmap[branchname] = branchheads
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield branchmap
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 except TypeError:
self._abort(error.ResponseError(_("unexpected response:"), d))
def branches(self, nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = encodelist(nodes)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 d = self._call("branches", nodes=n)
try:
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 br = [tuple(decodelist(b)) for b in d.splitlines()]
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 return br
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 self._abort(error.ResponseError(_("unexpected response:"), d))
def between(self, pairs):
Matt Mackall
protocol: unify basic http client requests
r11587 batch = 8 # avoid giant requests
r = []
for i in xrange(0, len(pairs), batch):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = " ".join([encodelist(p, '-') for p in pairs[i:i + batch]])
Matt Mackall
protocol: unify basic http client requests
r11587 d = self._call("between", pairs=n)
try:
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.extend(l and decodelist(l) or [] for l in d.splitlines())
Matt Mackall
wireproto: avoid naked excepts
r13726 except ValueError:
Matt Mackall
protocol: unify basic http client requests
r11587 self._abort(error.ResponseError(_("unexpected response:"), d))
return r
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def pushkey(self, namespace, key, old, new):
if not self.capable('pushkey'):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield False, None
f = future()
Pierre-Yves David
pushkey: add more verbose debug output regarding pushkey...
r17293 self.ui.debug('preparing pushkey for "%s:%s"\n' % (namespace, key))
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'namespace': encoding.fromlocal(namespace),
'key': encoding.fromlocal(key),
'old': encoding.fromlocal(old),
'new': encoding.fromlocal(new)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Pierre-Yves David
wireproto: handle other server output in pushkey...
r15652 d, output = d.split('\n', 1)
David Soria Parra
wireproto: catch possible cast error in pushkey...
r13450 try:
d = bool(int(d))
except ValueError:
raise error.ResponseError(
_('push failed (unexpected response):'), d)
Pierre-Yves David
wireproto: handle other server output in pushkey...
r15652 for l in output.splitlines(True):
self.ui.status(_('remote: '), l)
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield d
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 @batchable
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 def listkeys(self, namespace):
if not self.capable('pushkey'):
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 yield {}, None
f = future()
Pierre-Yves David
pushkey: add more verbose debug output regarding pushkey...
r17293 self.ui.debug('preparing listkeys for "%s"\n' % namespace)
Augie Fackler
wireproto: remove todict() and use {} literals instead
r20671 yield {'namespace': encoding.fromlocal(namespace)}, f
Peter Arrenbrecht
wireproto: make a number of commands batchable...
r14623 d = f.value
Pierre-Yves David
wireproto: use pushkey.decodekey
r21653 yield pushkeymod.decodekeys(d)
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586
Matt Mackall
protocol: unify stream_out client code
r11588 def stream_out(self):
return self._callstream('stream_out')
Matt Mackall
protocol: unify client changegroup methods
r11591 def changegroup(self, nodes, kind):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 n = encodelist(nodes)
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("changegroup", roots=n)
return changegroupmod.unbundle10(f, 'UN')
Matt Mackall
protocol: unify client changegroup methods
r11591
def changegroupsubset(self, bases, heads, kind):
self.requirecap('changegroupsubset', _('look up remote changes'))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 bases = encodelist(bases)
heads = encodelist(heads)
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("changegroupsubset",
bases=bases, heads=heads)
return changegroupmod.unbundle10(f, 'UN')
Matt Mackall
protocol: unify client changegroup methods
r11591
Pierre-Yves David
getbundle: declare type of parameters...
r21646 def getbundle(self, source, **kwargs):
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 self.requirecap('getbundle', _('look up remote changes'))
opts = {}
Pierre-Yves David
getbundle: declare type of parameters...
r21646 for key, value in kwargs.iteritems():
if value is None:
continue
keytype = gboptsmap.get(key)
if keytype is None:
assert False, 'unexpected'
elif keytype == 'nodes':
value = encodelist(value)
elif keytype == 'csv':
value = ','.join(value)
elif keytype != 'plain':
raise KeyError('unknown getbundle option type %s'
% keytype)
opts[key] = value
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 f = self._callcompressable("getbundle", **opts)
Pierre-Yves David
getbundle: declare type of parameters...
r21646 bundlecaps = kwargs.get('bundlecaps')
Pierre-Yves David
bundle2: use HG2X in the header...
r21144 if bundlecaps is not None and 'HG2X' in bundlecaps:
Pierre-Yves David
bundle2: allow bundle2 for pulling over the wire...
r21069 return bundle2.unbundle20(self.ui, f)
else:
return changegroupmod.unbundle10(f, 'UN')
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741
Matt Mackall
protocol: unify client unbundle support...
r11592 def unbundle(self, cg, heads, source):
'''Send cg (a readable file-like object representing the
changegroup to push, typically a chunkbuffer object) to the
Pierre-Yves David
bundle2: support for push over the wire...
r21075 remote server as a bundle.
When pushing a bundle10 stream, return an integer indicating the
result of the push (see localrepository.addchangegroup()).
When pushing a bundle20 stream, return a bundle20 stream.'''
Matt Mackall
protocol: unify client unbundle support...
r11592
Martin Geisler
wireproto: do not hash when heads == ['force']...
r14419 if heads != ['force'] and self.capable('unbundlehash'):
Shuhei Takahashi
wireproto: allow unbundle with hashed heads parameter (issue2126)...
r13942 heads = encodelist(['hashed',
util.sha1(''.join(sorted(heads))).digest()])
else:
heads = encodelist(heads)
Pierre-Yves David
bundle2: support for push over the wire...
r21075 if util.safehasattr(cg, 'deltaheader'):
# this a bundle10, do the old style call sequence
ret, output = self._callpush("unbundle", cg, heads=heads)
if ret == "":
raise error.ResponseError(
_('push failed:'), output)
try:
ret = int(ret)
except ValueError:
raise error.ResponseError(
_('push failed (unexpected response):'), ret)
Matt Mackall
protocol: unify client unbundle support...
r11592
Pierre-Yves David
bundle2: support for push over the wire...
r21075 for l in output.splitlines(True):
self.ui.status(_('remote: '), l)
else:
# bundle2 push. Send a stream, fetch a stream.
stream = self._calltwowaystream('unbundle', cg, heads=heads)
ret = bundle2.unbundle20(self.ui, stream)
Matt Mackall
protocol: unify client unbundle support...
r11592 return ret
Peter Arrenbrecht
wireproto: add test for new optional arg missing on server...
r14048 def debugwireargs(self, one, two, three=None, four=None, five=None):
Peter Arrenbrecht
debug: add debugwireargs to test argument passing over the wire...
r13720 # don't pass optional arguments left at their default value
opts = {}
if three is not None:
opts['three'] = three
if four is not None:
opts['four'] = four
return self._call('debugwireargs', one=one, two=two, **opts)
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _call(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a simple string.
returns the server reply as a string."""
raise NotImplementedError()
def _callstream(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a stream.
returns the server reply as a file like object."""
raise NotImplementedError()
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 def _callcompressable(self, cmd, **args):
"""execute <cmd> on the server
The command is expected to return a stream.
Mads Kiilerich
spelling: fixes from spell checker
r21024 The stream may have been compressed in some implementations. This
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 function takes care of the decompression. This is the only difference
with _callstream.
returns the server reply as a file like object.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _callpush(self, cmd, fp, **args):
"""execute a <cmd> on server
The command is expected to be related to a push. Push has a special
return method.
returns the server reply as a (ret, output) tuple. ret is either
empty (error) or a stringified int.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: add a _calltwowaystream method on wirepeer...
r21072 def _calltwowaystream(self, cmd, fp, **args):
"""execute <cmd> on server
The command will send a stream to the server and get a stream in reply.
"""
raise NotImplementedError()
Pierre-Yves David
wireproto: document protocol specific function of wirepeer...
r20904 def _abort(self, exception):
"""clearly abort the wire protocol connection and raise the exception
"""
raise NotImplementedError()
Matt Mackall
protocol: move basic ssh client commands to wirerepository
r11586 # server side
Pierre-Yves David
wireproto: document possible return type...
r20902 # wire protocol command can either return a string or one of these classes.
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 class streamres(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: binary stream
The call was successful and the result is a stream.
Iterate on the `self.gen` attribute to retrieve chunks.
"""
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 def __init__(self, gen):
self.gen = gen
class pushres(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: success with simple integer return
The call was successful and returned an integer contained in `self.res`.
"""
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 def __init__(self, res):
self.res = res
Benoit Boissinot
wireproto: introduce pusherr() to deal with "unsynced changes" error...
r12703 class pusherr(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: failure
The call failed. The `self.res` attribute contains the error message.
"""
Benoit Boissinot
wireproto: introduce pusherr() to deal with "unsynced changes" error...
r12703 def __init__(self, res):
self.res = res
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 class ooberror(object):
Pierre-Yves David
wireproto: document possible return type...
r20902 """wireproto reply: failure of a batch of operation
Something failed during a batch call. The error message is stored in
`self.message`.
"""
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 def __init__(self, message):
self.message = message
Matt Mackall
protocol: introduce wireproto.py...
r11581 def dispatch(repo, proto, command):
Kevin Bullock
filtering: rename filters to their antonyms...
r18382 repo = repo.filtered("served")
Matt Mackall
protocol: introduce wireproto.py...
r11581 func, spec = commands[command]
args = proto.getargs(spec)
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return func(repo, proto, *args)
Matt Mackall
protocol: introduce wireproto.py...
r11581
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 def options(cmd, keys, others):
opts = {}
for k in keys:
if k in others:
opts[k] = others[k]
del others[k]
if others:
Pierre-Yves David
wireproto: rephrase the error message for unknown getbundle parameters...
r21728 sys.stderr.write("warning: %s ignored unexpected arguments %s\n"
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 % (cmd, ",".join(others)))
return opts
Pierre-Yves David
wireproto: add decorator for wire protocol command...
r20906 # list of commands
commands = {}
def wireprotocommand(name, args=''):
Mads Kiilerich
spelling: fixes from spell checker
r21024 """decorator for wire protocol command"""
Pierre-Yves David
wireproto: add decorator for wire protocol command...
r20906 def register(func):
commands[name] = (func, args)
return func
return register
Pierre-Yves David
wireproto: use decorator for the batch command
r20907 @wireprotocommand('batch', 'cmds *')
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 def batch(repo, proto, cmds, others):
Kevin Bullock
filtering: rename filters to their antonyms...
r18382 repo = repo.filtered("served")
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 res = []
for pair in cmds.split(';'):
op, args = pair.split(' ', 1)
vals = {}
for a in args.split(','):
if a:
n, v = a.split('=')
vals[n] = unescapearg(v)
func, spec = commands[op]
if spec:
keys = spec.split()
data = {}
for k in keys:
if k == '*':
star = {}
for key in vals.keys():
if key not in keys:
star[key] = vals[key]
data['*'] = star
else:
data[k] = vals[k]
result = func(repo, proto, *[data[k] for k in keys])
else:
result = func(repo, proto)
Andrew Pritchard
wireproto: add out-of-band error class to allow remote repo to report errors...
r15017 if isinstance(result, ooberror):
return result
Peter Arrenbrecht
wireproto: add batching support to wirerepository...
r14622 res.append(escapearg(result))
return ';'.join(res)
Pierre-Yves David
wireproto: use decorator for the between command
r20908 @wireprotocommand('between', 'pairs')
Matt Mackall
protocol: add proto to method prototypes
r11583 def between(repo, proto, pairs):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 pairs = [decodelist(p, '-') for p in pairs.split(" ")]
Matt Mackall
protocol: introduce wireproto.py...
r11581 r = []
for b in repo.between(pairs):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.append(encodelist(b) + "\n")
Matt Mackall
protocol: introduce wireproto.py...
r11581 return "".join(r)
Pierre-Yves David
wireproto: use decorator for the branchmap command
r20909 @wireprotocommand('branchmap')
Matt Mackall
protocol: add proto to method prototypes
r11583 def branchmap(repo, proto):
Pierre-Yves David
clfilter: drop extra filtering in wireprotocol...
r18281 branchmap = repo.branchmap()
Matt Mackall
protocol: introduce wireproto.py...
r11581 heads = []
for branch, nodes in branchmap.iteritems():
Matt Mackall
branch: operate on branch names in local string space where possible...
r13047 branchname = urllib.quote(encoding.fromlocal(branch))
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 branchnodes = encodelist(nodes)
heads.append('%s %s' % (branchname, branchnodes))
Matt Mackall
protocol: introduce wireproto.py...
r11581 return '\n'.join(heads)
Pierre-Yves David
wireproto: use decorator for the branches command
r20910 @wireprotocommand('branches', 'nodes')
Matt Mackall
protocol: add proto to method prototypes
r11583 def branches(repo, proto, nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 nodes = decodelist(nodes)
Matt Mackall
protocol: introduce wireproto.py...
r11581 r = []
for b in repo.branches(nodes):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 r.append(encodelist(b) + "\n")
Matt Mackall
protocol: introduce wireproto.py...
r11581 return "".join(r)
Pierre-Yves David
wireproto: extract capabilities list in outside the wireproto function...
r20774
wireprotocaps = ['lookup', 'changegroupsubset', 'branchmap', 'pushkey',
'known', 'getbundle', 'unbundlehash', 'batch']
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775
def _capabilities(repo, proto):
"""return a list of capabilities for a repo
This function exists to allow extensions to easily wrap capabilities
computation
- returns a lists: easy to alter
- change done here will be propagated to both `capabilities` and `hello`
Mads Kiilerich
spelling: fixes from spell checker
r21024 command without any other action needed.
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 """
Pierre-Yves David
wireproto: extract capabilities list in outside the wireproto function...
r20774 # copy to prevent modification of the global list
caps = list(wireprotocaps)
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 if _allowstream(repo.ui):
Benoit Allard
protocol: Add the stream-preferred capability...
r16361 if repo.ui.configbool('server', 'preferuncompressed', False):
caps.append('stream-preferred')
Sune Foldager
clone: only use stream when we understand the revlog format...
r12296 requiredformats = repo.requirements & repo.supportedformats
# if our local revlogs are just revlogv1, add 'stream' cap
if not requiredformats - set(('revlogv1',)):
caps.append('stream')
# otherwise, add 'streamreqs' detailing our local revlog format
else:
caps.append('streamreqs=%s' % ','.join(requiredformats))
Pierre-Yves David
bundle2: require both client and server to opt in...
r21148 if repo.ui.configbool('experimental', 'bundle2-exp', False):
Pierre-Yves David
bundle2: advertise bundle2 caps in server capabilities...
r21141 capsblob = bundle2.encodecaps(repo.bundle2caps)
Pierre-Yves David
bundle2: rename server capability to bundle2-exp...
r21145 caps.append('bundle2-exp=' + urllib.quote(capsblob))
Matt Mackall
protocol: unify server-side capabilities functions
r11594 caps.append('unbundle=%s' % ','.join(changegroupmod.bundlepriority))
Steven Brown
httprepo: long arguments support (issue2126)...
r14093 caps.append('httpheader=1024')
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 return caps
Mads Kiilerich
spelling: fixes from spell checker
r21024 # If you are writing an extension and consider wrapping this function. Wrap
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 # `_capabilities` instead.
Pierre-Yves David
wireproto: use decorator for the capabilities command
r20911 @wireprotocommand('capabilities')
Pierre-Yves David
wireproto: move wireproto capabilities computation in a subfunction...
r20775 def capabilities(repo, proto):
return ' '.join(_capabilities(repo, proto))
Matt Mackall
protocol: unify server-side capabilities functions
r11594
Pierre-Yves David
wireproto: use decorator for the changegroup command
r20912 @wireprotocommand('changegroup', 'roots')
Matt Mackall
protocol: unify changegroup commands...
r11584 def changegroup(repo, proto, roots):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 nodes = decodelist(roots)
Pierre-Yves David
localrepo: move the changegroup method in changegroup module...
r20931 cg = changegroupmod.changegroup(repo, nodes, 'serve')
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return streamres(proto.groupchunks(cg))
Matt Mackall
protocol: unify changegroup commands...
r11584
Pierre-Yves David
wireproto: use decorator for the changegroupsubset command
r20913 @wireprotocommand('changegroupsubset', 'bases heads')
Matt Mackall
protocol: unify changegroup commands...
r11584 def changegroupsubset(repo, proto, bases, heads):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 bases = decodelist(bases)
heads = decodelist(heads)
Pierre-Yves David
localrepo: move the changegroupsubset method in changegroup module...
r20927 cg = changegroupmod.changegroupsubset(repo, bases, heads, 'serve')
Dirkjan Ochtman
protocol: wrap non-string protocol responses in classes
r11625 return streamres(proto.groupchunks(cg))
Matt Mackall
protocol: unify changegroup commands...
r11584
Pierre-Yves David
wireproto: use decorator for the debugwireargs command
r20914 @wireprotocommand('debugwireargs', 'one two *')
Peter Arrenbrecht
wireproto: fix handling of '*' args for HTTP and SSH
r13721 def debugwireargs(repo, proto, one, two, others):
# only accept optional args from the known set
opts = options('debugwireargs', ['three', 'four'], others)
return repo.debugwireargs(one, two, **opts)
Peter Arrenbrecht
debug: add debugwireargs to test argument passing over the wire...
r13720
Pierre-Yves David
wireproto: expose the list of getbundle arguments to extensions...
r21615 # List of options accepted by getbundle.
#
# Meant to be extended by extensions. It is the extension's responsibility to
# ensure such options are properly processed in exchange.getbundle.
gboptslist = ['heads', 'common', 'bundlecaps']
Pierre-Yves David
wireproto: use decorator for the getbundle command
r20915 @wireprotocommand('getbundle', '*')
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 def getbundle(repo, proto, others):
Pierre-Yves David
getbundle: declare type of parameters...
r21646 opts = options('getbundle', gboptsmap.keys(), others)
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 for k, v in opts.iteritems():
Pierre-Yves David
getbundle: declare type of parameters...
r21646 keytype = gboptsmap[k]
if keytype == 'nodes':
Benoit Boissinot
bundle-ng: add bundlecaps argument to getbundle() command
r19201 opts[k] = decodelist(v)
Pierre-Yves David
getbundle: declare type of parameters...
r21646 elif keytype == 'csv':
Benoit Boissinot
bundle-ng: add bundlecaps argument to getbundle() command
r19201 opts[k] = set(v.split(','))
Pierre-Yves David
getbundle: declare type of parameters...
r21646 elif keytype != 'plain':
raise KeyError('unknown getbundle option type %s'
% keytype)
Pierre-Yves David
bundle2: allow bundle2 for pulling over the wire...
r21069 cg = exchange.getbundle(repo, 'serve', **opts)
Peter Arrenbrecht
wireproto: add getbundle() function...
r13741 return streamres(proto.groupchunks(cg))
Pierre-Yves David
wireproto: use decorator for the heads command
r20916 @wireprotocommand('heads')
Matt Mackall
protocol: add proto to method prototypes
r11583 def heads(repo, proto):
Pierre-Yves David
clfilter: drop extra filtering in wireprotocol...
r18281 h = repo.heads()
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 return encodelist(h) + "\n"
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: use decorator for the hello command
r20917 @wireprotocommand('hello')
Matt Mackall
protocol: unify server-side capabilities functions
r11594 def hello(repo, proto):
'''the hello command returns a set of lines describing various
interesting things about the server, in an RFC822-like format.
Currently the only one defined is "capabilities", which
consists of a line in the form:
capabilities: space separated list of tokens
'''
return "capabilities: %s\n" % (capabilities(repo, proto))
Pierre-Yves David
wireproto: use decorator for the listkeys command
r20919 @wireprotocommand('listkeys', 'namespace')
Matt Mackall
protocol: add proto to method prototypes
r11583 def listkeys(repo, proto, namespace):
Andreas Freimuth
wireproto: do not call pushkey module directly (issue3041)...
r15217 d = repo.listkeys(encoding.tolocal(namespace)).items()
Pierre-Yves David
wireproto: use pushkey.encodekey
r21651 return pushkeymod.encodekeys(d)
Matt Mackall
protocol: introduce wireproto.py...
r11581
Pierre-Yves David
wireproto: use decorator for the lookup command
r20920 @wireprotocommand('lookup', 'key')
Matt Mackall
protocol: add proto to method prototypes
r11583 def lookup(repo, proto, key):
Matt Mackall
protocol: introduce wireproto.py...
r11581 try:
Matt Mackall
wireproto: refuse to lookup secret csets
r15925 k = encoding.tolocal(key)
c = repo[k]
r = c.hex()
Matt Mackall
protocol: introduce wireproto.py...
r11581 success = 1
except Exception, inst:
r = str(inst)
success = 0
return "%s %s\n" % (success, r)
Pierre-Yves David
wireproto: use decorator for the known command
r20918 @wireprotocommand('known', 'nodes *')
Peter Arrenbrecht
wireproto: enable optional args for known() for future extensibility...
r14436 def known(repo, proto, nodes, others):
Peter Arrenbrecht
wireproto: add known([id]) function...
r13723 return ''.join(b and "1" or "0" for b in repo.known(decodelist(nodes)))
Pierre-Yves David
wireproto: use decorator for the pushkey command
r20921 @wireprotocommand('pushkey', 'namespace key old new')
Matt Mackall
protocol: add proto to method prototypes
r11583 def pushkey(repo, proto, namespace, key, old, new):
Matt Mackall
pushkey: use UTF-8
r13050 # compatibility with pre-1.8 clients which were accidentally
# sending raw binary nodes rather than utf-8-encoded hex
if len(new) == 20 and new.encode('string-escape') != new:
# looks like it could be a binary node
try:
Alexander Solovyov
remove unused imports and variables
r14064 new.decode('utf-8')
Matt Mackall
pushkey: use UTF-8
r13050 new = encoding.tolocal(new) # but cleanly decodes as UTF-8
except UnicodeDecodeError:
pass # binary, leave unmodified
else:
new = encoding.tolocal(new) # normal path
Wagner Bruna
wireproto: fix pushkey hook failure and output on remote http repo...
r17793 if util.safehasattr(proto, 'restore'):
proto.redirect()
try:
r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key),
encoding.tolocal(old), new) or False
except util.Abort:
r = False
output = proto.restore()
return '%s\n%s' % (int(r), output)
Andreas Freimuth
wireproto: do not call pushkey module directly (issue3041)...
r15217 r = repo.pushkey(encoding.tolocal(namespace), encoding.tolocal(key),
encoding.tolocal(old), new)
Matt Mackall
protocol: introduce wireproto.py...
r11581 return '%s\n' % int(r)
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 def _allowstream(ui):
return ui.configbool('server', 'uncompressed', True, untrusted=True)
Durham Goode
clone: move file stream walk to a separate function...
r19176 def _walkstreamfiles(repo):
# this is it's own function so extensions can override it
return repo.store.walk()
Pierre-Yves David
wireproto: use decorator for the stream command
r20922 @wireprotocommand('stream_out')
Matt Mackall
protocol: unify stream_out command
r11585 def stream(repo, proto):
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 '''If the server supports streaming clone, it advertises the "stream"
capability with a value representing the version and flags of the repo
it is serving. Client checks to see if it understands the format.
The format is simple: the server writes out a line with the amount
Mads Kiilerich
fix trivial spelling errors
r17424 of files, then the total amount of bytes to be transferred (separated
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 by a space). Then, for each file, the server first writes the filename
Mads Kiilerich
spelling: fixes from spell checker
r21024 and file size (separated by the null character), then the file contents.
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 '''
if not _allowstream(repo.ui):
return '1\n'
entries = []
total_bytes = 0
try:
# get consistent snapshot of repo, lock during scan
lock = repo.lock()
try:
repo.ui.debug('scanning\n')
Durham Goode
clone: move file stream walk to a separate function...
r19176 for name, ename, size in _walkstreamfiles(repo):
Mads Kiilerich
clone: don't include empty revlogs in stream
r18381 if size:
entries.append((name, size))
total_bytes += size
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627 finally:
lock.release()
except error.LockError:
return '2\n' # error: 2
def streamer(repo, entries, total):
'''stream out all metadata files in repository.'''
yield '0\n' # success
repo.ui.debug('%d files, %d bytes to transfer\n' %
(len(entries), total_bytes))
yield '%d %d\n' % (len(entries), total_bytes)
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556
sopener = repo.sopener
oldaudit = sopener.mustaudit
Bryan O'Sullivan
wireproto: don't format a debug string inside a hot loop...
r17558 debugflag = repo.ui.debugflag
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 sopener.mustaudit = False
try:
for name, size in entries:
Bryan O'Sullivan
wireproto: don't format a debug string inside a hot loop...
r17558 if debugflag:
repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 # partially encode name over the wire for backwards compat
yield '%s\0%d\n' % (store.encodedir(name), size)
Bryan O'Sullivan
wireproto: bypass filechunkiter for small files when streaming...
r17557 if size <= 65536:
Patrick Mezard
wireproto: fix check-code.py breakage introduced by 6d97dd630d79
r17567 fp = sopener(name)
try:
data = fp.read(size)
finally:
fp.close()
yield data
Bryan O'Sullivan
wireproto: bypass filechunkiter for small files when streaming...
r17557 else:
for chunk in util.filechunkiter(sopener(name), limit=size):
yield chunk
Thomas Arendsen Hein
wireproto: workaround for yield inside try/finally incompatible with python2.4
r17603 # replace with "finally:" when support for python 2.4 has been dropped
except Exception:
Bryan O'Sullivan
wireproto: don't audit local paths during stream_out...
r17556 sopener.mustaudit = oldaudit
Thomas Arendsen Hein
wireproto: workaround for yield inside try/finally incompatible with python2.4
r17603 raise
sopener.mustaudit = oldaudit
Dirkjan Ochtman
protocol: move the streamclone implementation into wireproto
r11627
return streamres(streamer(repo, entries, total_bytes))
Matt Mackall
protocol: unify stream_out command
r11585
Pierre-Yves David
wireproto: use decorator for the ubundle command
r20923 @wireprotocommand('unbundle', 'heads')
Matt Mackall
protocol: unify unbundle on the server side
r11593 def unbundle(repo, proto, heads):
Benoit Boissinot
wireproto: refactor list of nodeid encoding / decoding
r11597 their_heads = decodelist(heads)
Matt Mackall
protocol: unify unbundle on the server side
r11593
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 try:
proto.redirect()
Matt Mackall
protocol: unify unbundle on the server side
r11593
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 exchange.check_heads(repo, their_heads, 'preparing changes')
Benoit Boissinot
wireproto: redirect the output earlier
r12702
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 # write bundle data to temporary file because it can be big
fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
fp = os.fdopen(fd, 'wb+')
r = 0
Matt Mackall
protocol: unify unbundle on the server side
r11593 try:
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 proto.getfile(fp)
Pierre-Yves David
unbundle: extract the core logic in another function...
r20968 fp.seek(0)
Pierre-Yves David
bundle2: add a ui argument to readbundle...
r21064 gen = exchange.readbundle(repo.ui, fp, None)
Pierre-Yves David
unbundle: extract the core logic in another function...
r20968 r = exchange.unbundle(repo, gen, their_heads, 'serve',
proto._client())
Pierre-Yves David
bundle2: support for push over the wire...
r21075 if util.safehasattr(r, 'addpart'):
# The return looks streameable, we are in the bundle2 case and
# should return a stream.
return streamres(r.getchunks())
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 return pushres(r)
Matt Mackall
protocol: unify unbundle on the server side
r11593 finally:
Pierre-Yves David
unbundle: extract checkheads in its own function...
r20967 fp.close()
os.unlink(tempname)
Pierre-Yves David
bundle2: move exception classes into the error module...
r21618 except error.BundleValueError, exc:
Pierre-Yves David
bundle2: gracefully handle UnknownPartError during unbundle...
r21183 bundler = bundle2.bundle20(repo.ui)
Pierre-Yves David
bundle2: rename b2x:error:unknownpart to b2x:error:unsupportedcontent...
r21619 errpart = bundler.newpart('B2X:ERROR:UNSUPPORTEDCONTENT')
Pierre-Yves David
bundle2: support None parttype in BundleValueError...
r21627 if exc.parttype is not None:
errpart.addparam('parttype', exc.parttype)
Pierre-Yves David
bundle2: support transmission of params error over the wire...
r21622 if exc.params:
errpart.addparam('params', '\0'.join(exc.params))
Pierre-Yves David
bundle2: gracefully handle UnknownPartError during unbundle...
r21183 return streamres(bundler.getchunks())
Pierre-Yves David
localrepo: add unbundle support...
r20969 except util.Abort, inst:
# The old code we moved used sys.stderr directly.
Mads Kiilerich
spelling: fixes from spell checker
r21024 # We did not change it to minimise code change.
Pierre-Yves David
localrepo: add unbundle support...
r20969 # This need to be moved to something proper.
# Feel free to do it.
Pierre-Yves David
bundle2: gracefully handle abort during unbundle...
r21177 if getattr(inst, 'duringunbundle2', False):
bundler = bundle2.bundle20(repo.ui)
manargs = [('message', str(inst))]
advargs = []
if inst.hint is not None:
advargs.append(('hint', inst.hint))
bundler.addpart(bundle2.bundlepart('B2X:ERROR:ABORT',
manargs, advargs))
return streamres(bundler.getchunks())
else:
sys.stderr.write("abort: %s\n" % inst)
return pushres(0)
Pierre-Yves David
bundle2: fix raising errors during heads checking...
r21184 except error.PushRaced, exc:
Pierre-Yves David
bundle2: gracefully handle PushRaced error during unbundle...
r21186 if getattr(exc, 'duringunbundle2', False):
bundler = bundle2.bundle20(repo.ui)
Pierre-Yves David
bundle2: update all ``addpart`` callers to ``newpart``...
r21600 bundler.newpart('B2X:ERROR:PUSHRACED', [('message', str(exc))])
Pierre-Yves David
bundle2: gracefully handle PushRaced error during unbundle...
r21186 return streamres(bundler.getchunks())
else:
return pusherr(str(exc))