httppeer.py
1115 lines
| 34.1 KiB
| text/x-python
|
PythonLexer
/ mercurial / httppeer.py
|
r17192 | # httppeer.py - HTTP repository proxy classes for mercurial | ||
# | ||||
# Copyright 2005, 2006 Matt Mackall <mpm@selenic.com> | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r25954 | from __future__ import absolute_import | ||
import errno | ||||
|
r33820 | import io | ||
|
r25954 | import os | ||
import socket | ||||
|
r30763 | import struct | ||
|
r37669 | import weakref | ||
|
r25954 | |||
from .i18n import _ | ||||
|
r43359 | from .pycompat import getattr | ||
|
r25954 | from . import ( | ||
|
r28666 | bundle2, | ||
|
r25954 | error, | ||
httpconnection, | ||||
|
r30924 | pycompat, | ||
|
r25954 | statichttprepo, | ||
|
r36977 | url as urlmod, | ||
|
r25954 | util, | ||
|
r37501 | wireprotoframing, | ||
|
r37576 | wireprototypes, | ||
|
r37632 | wireprotov1peer, | ||
|
r37737 | wireprotov2peer, | ||
|
r37563 | wireprotov2server, | ||
|
r25954 | ) | ||
|
r43079 | from .interfaces import ( | ||
repository, | ||||
util as interfaceutil, | ||||
) | ||||
|
r37828 | from .utils import ( | ||
|
r39476 | cborutil, | ||
|
r39467 | stringutil, | ||
|
r37828 | ) | ||
|
r17192 | |||
|
r29455 | httplib = util.httplib | ||
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
|
r43346 | |||
|
r30759 | def encodevalueinheaders(value, header, limit): | ||
"""Encode a string value into multiple HTTP headers. | ||||
``value`` will be encoded into 1 or more HTTP headers with the names | ||||
``header-<N>`` where ``<N>`` is an integer starting at 1. Each header | ||||
name + value will be at most ``limit`` bytes long. | ||||
|
r34733 | Returns an iterable of 2-tuples consisting of header names and | ||
values as native strings. | ||||
|
r30759 | """ | ||
|
r34733 | # HTTP Headers are ASCII. Python 3 requires them to be unicodes, | ||
# not bytes. This function always takes bytes in as arguments. | ||||
fmt = pycompat.strurl(header) + r'-%s' | ||||
# Note: it is *NOT* a bug that the last bit here is a bytestring | ||||
# and not a unicode: we're just getting the encoded length anyway, | ||||
# and using an r-string to make it portable between Python 2 and 3 | ||||
# doesn't work because then the \r is a literal backslash-r | ||||
# instead of a carriage return. | ||||
|
r43347 | valuelen = limit - len(fmt % r'000') - len(b': \r\n') | ||
|
r30759 | result = [] | ||
n = 0 | ||||
|
r38806 | for i in pycompat.xrange(0, len(value), valuelen): | ||
|
r30759 | n += 1 | ||
|
r43346 | result.append((fmt % str(n), pycompat.strurl(value[i : i + valuelen]))) | ||
|
r30759 | |||
return result | ||||
|
r43346 | |||
|
r33820 | class _multifile(object): | ||
def __init__(self, *fileobjs): | ||||
for f in fileobjs: | ||||
|
r43347 | if not util.safehasattr(f, b'length'): | ||
|
r33820 | raise ValueError( | ||
|
r43347 | b'_multifile only supports file objects that ' | ||
b'have a length but this one does not:', | ||||
|
r43346 | type(f), | ||
f, | ||||
) | ||||
|
r33820 | self._fileobjs = fileobjs | ||
self._index = 0 | ||||
@property | ||||
def length(self): | ||||
return sum(f.length for f in self._fileobjs) | ||||
def read(self, amt=None): | ||||
if amt <= 0: | ||||
|
r43347 | return b''.join(f.read() for f in self._fileobjs) | ||
|
r33820 | parts = [] | ||
while amt and self._index < len(self._fileobjs): | ||||
parts.append(self._fileobjs[self._index].read(amt)) | ||||
got = len(parts[-1]) | ||||
if got < amt: | ||||
self._index += 1 | ||||
amt -= got | ||||
|
r43347 | return b''.join(parts) | ||
|
r33820 | |||
def seek(self, offset, whence=os.SEEK_SET): | ||||
if whence != os.SEEK_SET: | ||||
raise NotImplementedError( | ||||
|
r43347 | b'_multifile does not support anything other' | ||
b' than os.SEEK_SET for whence on seek()' | ||||
|
r43346 | ) | ||
|
r33820 | if offset != 0: | ||
raise NotImplementedError( | ||||
|
r43347 | b'_multifile only supports seeking to start, but that ' | ||
b'could be fixed if you need it' | ||||
|
r43346 | ) | ||
|
r33820 | for f in self._fileobjs: | ||
f.seek(0) | ||||
self._index = 0 | ||||
|
r43346 | |||
def makev1commandrequest( | ||||
ui, requestbuilder, caps, capablefn, repobaseurl, cmd, args | ||||
): | ||||
|
r37567 | """Make an HTTP request to run a command for a version 1 client. | ||
``caps`` is a set of known server capabilities. The value may be | ||||
None if capabilities are not yet known. | ||||
``capablefn`` is a function to evaluate a capability. | ||||
``cmd``, ``args``, and ``data`` define the command, its arguments, and | ||||
raw data to pass to it. | ||||
""" | ||||
|
r43347 | if cmd == b'pushkey': | ||
args[b'data'] = b'' | ||||
data = args.pop(b'data', None) | ||||
headers = args.pop(b'headers', {}) | ||||
|
r37567 | |||
|
r43347 | ui.debug(b"sending %s command\n" % cmd) | ||
q = [(b'cmd', cmd)] | ||||
|
r37567 | headersize = 0 | ||
# Important: don't use self.capable() here or else you end up | ||||
# with infinite recursion when trying to look up capabilities | ||||
# for the first time. | ||||
|
r43347 | postargsok = caps is not None and b'httppostargs' in caps | ||
|
r37567 | |||
# Send arguments via POST. | ||||
if postargsok and args: | ||||
strargs = urlreq.urlencode(sorted(args.items())) | ||||
if not data: | ||||
data = strargs | ||||
else: | ||||
if isinstance(data, bytes): | ||||
i = io.BytesIO(data) | ||||
i.length = len(data) | ||||
data = i | ||||
argsio = io.BytesIO(strargs) | ||||
argsio.length = len(strargs) | ||||
data = _multifile(argsio, data) | ||||
headers[r'X-HgArgs-Post'] = len(strargs) | ||||
elif args: | ||||
# Calling self.capable() can infinite loop if we are calling | ||||
# "capabilities". But that command should never accept wire | ||||
# protocol arguments. So this should never happen. | ||||
|
r43347 | assert cmd != b'capabilities' | ||
httpheader = capablefn(b'httpheader') | ||||
|
r37567 | if httpheader: | ||
|
r43347 | headersize = int(httpheader.split(b',', 1)[0]) | ||
|
r37567 | |||
# Send arguments via HTTP headers. | ||||
if headersize > 0: | ||||
# The headers can typically carry more data than the URL. | ||||
encargs = urlreq.urlencode(sorted(args.items())) | ||||
|
r43346 | for header, value in encodevalueinheaders( | ||
|
r43347 | encargs, b'X-HgArg', headersize | ||
|
r43346 | ): | ||
|
r37567 | headers[header] = value | ||
# Send arguments via query string (Mercurial <1.9). | ||||
else: | ||||
q += sorted(args.items()) | ||||
|
r43347 | qs = b'?%s' % urlreq.urlencode(q) | ||
cu = b"%s%s" % (repobaseurl, qs) | ||||
|
r37567 | size = 0 | ||
|
r43347 | if util.safehasattr(data, b'length'): | ||
|
r37567 | size = data.length | ||
elif data is not None: | ||||
size = len(data) | ||||
if data is not None and r'Content-Type' not in headers: | ||||
headers[r'Content-Type'] = r'application/mercurial-0.1' | ||||
# Tell the server we accept application/mercurial-0.2 and multiple | ||||
# compression formats if the server is capable of emitting those | ||||
# payloads. | ||||
|
r37574 | # Note: Keep this set empty by default, as client advertisement of | ||
# protocol parameters should only occur after the handshake. | ||||
protoparams = set() | ||||
|
r37567 | |||
mediatypes = set() | ||||
if caps is not None: | ||||
|
r43347 | mt = capablefn(b'httpmediatype') | ||
|
r37567 | if mt: | ||
|
r43347 | protoparams.add(b'0.1') | ||
mediatypes = set(mt.split(b',')) | ||||
|
r37567 | |||
|
r43347 | protoparams.add(b'partial-pull') | ||
|
r37574 | |||
|
r43347 | if b'0.2tx' in mediatypes: | ||
protoparams.add(b'0.2') | ||||
|
r37567 | |||
|
r43347 | if b'0.2tx' in mediatypes and capablefn(b'compression'): | ||
|
r37567 | # We /could/ compare supported compression formats and prune | ||
# non-mutually supported or error if nothing is mutually supported. | ||||
# For now, send the full list to the server and have it error. | ||||
|
r43346 | comps = [ | ||
e.wireprotosupport().name | ||||
for e in util.compengines.supportedwireengines(util.CLIENTROLE) | ||||
] | ||||
|
r43347 | protoparams.add(b'comp=%s' % b','.join(comps)) | ||
|
r37567 | |||
if protoparams: | ||||
|
r43346 | protoheaders = encodevalueinheaders( | ||
|
r43347 | b' '.join(sorted(protoparams)), b'X-HgProto', headersize or 1024 | ||
|
r43346 | ) | ||
|
r37567 | for header, value in protoheaders: | ||
headers[header] = value | ||||
|
r37573 | |||
varyheaders = [] | ||||
for header in headers: | ||||
if header.lower().startswith(r'x-hg'): | ||||
|
r37567 | varyheaders.append(header) | ||
if varyheaders: | ||||
|
r37573 | headers[r'Vary'] = r','.join(sorted(varyheaders)) | ||
|
r37567 | |||
req = requestbuilder(pycompat.strurl(cu), data, headers) | ||||
if data is not None: | ||||
|
r43347 | ui.debug(b"sending %d bytes\n" % size) | ||
|
r37567 | req.add_unredirected_header(r'Content-Length', r'%d' % size) | ||
return req, cu, qs | ||||
|
r43346 | |||
|
r37756 | def _reqdata(req): | ||
"""Get request data, if any. If no data, returns None.""" | ||||
if pycompat.ispy3: | ||||
return req.data | ||||
if not req.has_data(): | ||||
return None | ||||
return req.get_data() | ||||
|
r43346 | |||
|
r37566 | def sendrequest(ui, opener, req): | ||
"""Send a prepared HTTP request. | ||||
Returns the response object. | ||||
""" | ||||
|
r38139 | dbg = ui.debug | ||
|
r43347 | if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'): | ||
line = b'devel-peer-request: %s\n' | ||||
|
r43346 | dbg( | ||
line | ||||
|
r43347 | % b'%s %s' | ||
|
r43346 | % ( | ||
pycompat.bytesurl(req.get_method()), | ||||
pycompat.bytesurl(req.get_full_url()), | ||||
) | ||||
) | ||||
|
r37566 | hgargssize = None | ||
for header, value in sorted(req.header_items()): | ||||
|
r37755 | header = pycompat.bytesurl(header) | ||
value = pycompat.bytesurl(value) | ||||
|
r43347 | if header.startswith(b'X-hgarg-'): | ||
|
r37566 | if hgargssize is None: | ||
hgargssize = 0 | ||||
hgargssize += len(value) | ||||
else: | ||||
|
r43347 | dbg(line % b' %s %s' % (header, value)) | ||
|
r37566 | |||
if hgargssize is not None: | ||||
|
r43346 | dbg( | ||
line | ||||
|
r43347 | % b' %d bytes of commands arguments in headers' | ||
|
r43346 | % hgargssize | ||
) | ||||
|
r37756 | data = _reqdata(req) | ||
if data is not None: | ||||
|
r37566 | length = getattr(data, 'length', None) | ||
if length is None: | ||||
length = len(data) | ||||
|
r43347 | dbg(line % b' %d bytes of data' % length) | ||
|
r37566 | |||
start = util.timer() | ||||
|
r38521 | res = None | ||
|
r37568 | try: | ||
res = opener.open(req) | ||||
except urlerr.httperror as inst: | ||||
if inst.code == 401: | ||||
|
r43347 | raise error.Abort(_(b'authorization failed')) | ||
|
r37568 | raise | ||
except httplib.HTTPException as inst: | ||||
|
r43346 | ui.debug( | ||
|
r43347 | b'http error requesting %s\n' | ||
% util.hidepassword(req.get_full_url()) | ||||
|
r43346 | ) | ||
|
r37568 | ui.traceback() | ||
raise IOError(None, inst) | ||||
finally: | ||||
|
r43347 | if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'): | ||
|
r38521 | code = res.code if res else -1 | ||
|
r43346 | dbg( | ||
line | ||||
|
r43347 | % b' finished in %.4f seconds (%d)' | ||
|
r43346 | % (util.timer() - start, code) | ||
) | ||||
|
r37568 | |||
# Insert error handlers for common I/O failures. | ||||
|
r40054 | urlmod.wrapresponse(res) | ||
|
r37566 | |||
return res | ||||
|
r43346 | |||
|
r37851 | class RedirectedRepoError(error.RepoError): | ||
def __init__(self, msg, respurl): | ||||
super(RedirectedRepoError, self).__init__(msg) | ||||
self.respurl = respurl | ||||
|
r43346 | |||
def parsev1commandresponse( | ||||
ui, baseurl, requrl, qs, resp, compressible, allowcbor=False | ||||
): | ||||
|
r37569 | # record the url we got redirected to | ||
|
r37851 | redirected = False | ||
|
r37569 | respurl = pycompat.bytesurl(resp.geturl()) | ||
if respurl.endswith(qs): | ||||
|
r43346 | respurl = respurl[: -len(qs)] | ||
|
r37851 | qsdropped = False | ||
else: | ||||
qsdropped = True | ||||
|
r43347 | if baseurl.rstrip(b'/') != respurl.rstrip(b'/'): | ||
|
r37851 | redirected = True | ||
|
r37569 | if not ui.quiet: | ||
|
r43347 | ui.warn(_(b'real URL is %s\n') % respurl) | ||
|
r37569 | |||
try: | ||||
proto = pycompat.bytesurl(resp.getheader(r'content-type', r'')) | ||||
except AttributeError: | ||||
proto = pycompat.bytesurl(resp.headers.get(r'content-type', r'')) | ||||
safeurl = util.hidepassword(baseurl) | ||||
|
r43347 | if proto.startswith(b'application/hg-error'): | ||
|
r37569 | raise error.OutOfBandError(resp.read()) | ||
|
r37572 | |||
# Pre 1.0 versions of Mercurial used text/plain and | ||||
# application/hg-changegroup. We don't support such old servers. | ||||
|
r43347 | if not proto.startswith(b'application/mercurial-'): | ||
ui.debug(b"requested URL: '%s'\n" % util.hidepassword(requrl)) | ||||
|
r43346 | msg = _( | ||
|
r43347 | b"'%s' does not appear to be an hg repository:\n" | ||
b"---%%<--- (%s)\n%s\n---%%<---\n" | ||||
) % (safeurl, proto or b'no content-type', resp.read(1024)) | ||||
|
r37851 | |||
# Some servers may strip the query string from the redirect. We | ||||
# raise a special error type so callers can react to this specially. | ||||
if redirected and qsdropped: | ||||
raise RedirectedRepoError(msg, respurl) | ||||
else: | ||||
raise error.RepoError(msg) | ||||
|
r37569 | |||
|
r37572 | try: | ||
|
r43347 | subtype = proto.split(b'-', 1)[1] | ||
|
r37576 | |||
# Unless we end up supporting CBOR in the legacy wire protocol, | ||||
# this should ONLY be encountered for the initial capabilities | ||||
# request during handshake. | ||||
|
r43347 | if subtype == b'cbor': | ||
|
r37576 | if allowcbor: | ||
return respurl, proto, resp | ||||
else: | ||||
|
r43346 | raise error.RepoError( | ||
|
r43387 | _(b'unexpected CBOR response from server') | ||
|
r43346 | ) | ||
|
r37576 | |||
|
r43347 | version_info = tuple([int(n) for n in subtype.split(b'.')]) | ||
|
r37572 | except ValueError: | ||
|
r43346 | raise error.RepoError( | ||
|
r43387 | _(b"'%s' sent a broken Content-Type header (%s)") % (safeurl, proto) | ||
|
r43346 | ) | ||
|
r37569 | |||
|
r37572 | # TODO consider switching to a decompression reader that uses | ||
# generators. | ||||
if version_info == (0, 1): | ||||
if compressible: | ||||
|
r43347 | resp = util.compengines[b'zlib'].decompressorreader(resp) | ||
|
r37569 | |||
|
r37572 | elif version_info == (0, 2): | ||
# application/mercurial-0.2 always identifies the compression | ||||
# engine in the payload header. | ||||
|
r43347 | elen = struct.unpack(b'B', util.readexactly(resp, 1))[0] | ||
r39520 | ename = util.readexactly(resp, elen) | |||
|
r37572 | engine = util.compengines.forwiretype(ename) | ||
|
r37569 | |||
|
r37572 | resp = engine.decompressorreader(resp) | ||
else: | ||||
|
r43346 | raise error.RepoError( | ||
|
r43347 | _(b"'%s' uses newer protocol %s") % (safeurl, subtype) | ||
|
r43346 | ) | ||
|
r37569 | |||
|
r37576 | return respurl, proto, resp | ||
|
r37569 | |||
|
r43346 | |||
|
r37632 | class httppeer(wireprotov1peer.wirepeer): | ||
|
r37570 | def __init__(self, ui, path, url, opener, requestbuilder, caps): | ||
|
r37337 | self.ui = ui | ||
|
r33671 | self._path = path | ||
|
r37024 | self._url = url | ||
|
r37570 | self._caps = caps | ||
|
r43347 | self.limitedarguments = caps is not None and b'httppostargs' not in caps | ||
|
r37024 | self._urlopener = opener | ||
|
r37565 | self._requestbuilder = requestbuilder | ||
|
r17192 | |||
def __del__(self): | ||||
|
r37024 | for h in self._urlopener.handlers: | ||
h.close() | ||||
getattr(h, "close_all", lambda: None)() | ||||
|
r17192 | |||
|
r37336 | # Begin of ipeerconnection interface. | ||
|
r33804 | |||
|
r17192 | def url(self): | ||
|
r33671 | return self._path | ||
|
r17192 | |||
|
r33804 | def local(self): | ||
return None | ||||
def peer(self): | ||||
return self | ||||
def canpush(self): | ||||
return True | ||||
|
r17192 | |||
|
r33804 | def close(self): | ||
|
r40478 | try: | ||
|
r43346 | reqs, sent, recv = ( | ||
self._urlopener.requestscount, | ||||
self._urlopener.sentbytescount, | ||||
self._urlopener.receivedbytescount, | ||||
) | ||||
|
r40478 | except AttributeError: | ||
return | ||||
|
r43346 | self.ui.note( | ||
_( | ||||
|
r43347 | b'(sent %d HTTP requests and %d bytes; ' | ||
b'received %d bytes in responses)\n' | ||||
|
r43346 | ) | ||
% (reqs, sent, recv) | ||||
) | ||||
|
r17192 | |||
|
r37336 | # End of ipeerconnection interface. | ||
|
r33804 | |||
|
r37336 | # Begin of ipeercommands interface. | ||
|
r33804 | |||
def capabilities(self): | ||||
|
r33671 | return self._caps | ||
|
r17192 | |||
|
r37336 | # End of ipeercommands interface. | ||
|
r33804 | |||
|
r30464 | def _callstream(self, cmd, _compressible=False, **args): | ||
|
r35360 | args = pycompat.byteskwargs(args) | ||
|
r36236 | |||
|
r43346 | req, cu, qs = makev1commandrequest( | ||
self.ui, | ||||
self._requestbuilder, | ||||
self._caps, | ||||
self.capable, | ||||
self._url, | ||||
cmd, | ||||
args, | ||||
) | ||||
|
r30763 | |||
|
r37568 | resp = sendrequest(self.ui, self._urlopener, req) | ||
|
r32002 | |||
|
r43346 | self._url, ct, resp = parsev1commandresponse( | ||
self.ui, self._url, cu, qs, resp, _compressible | ||||
) | ||||
|
r30464 | |||
|
r17192 | return resp | ||
def _call(self, cmd, **args): | ||||
fp = self._callstream(cmd, **args) | ||||
try: | ||||
return fp.read() | ||||
finally: | ||||
# if using keepalive, allow connection to be reused | ||||
fp.close() | ||||
def _callpush(self, cmd, cg, **args): | ||||
# have to stream bundle to a temp file because we do not have | ||||
# http 1.1 chunked transfer. | ||||
|
r43347 | types = self.capable(b'unbundle') | ||
|
r17192 | try: | ||
|
r43347 | types = types.split(b',') | ||
|
r17192 | except AttributeError: | ||
# servers older than d1b16a746db6 will send 'unbundle' as a | ||||
# boolean capability. They only support headerless/uncompressed | ||||
# bundles. | ||||
|
r43347 | types = [b""] | ||
|
r17192 | for x in types: | ||
|
r28666 | if x in bundle2.bundletypes: | ||
|
r17192 | type = x | ||
break | ||||
|
r28666 | tempname = bundle2.writebundle(self.ui, cg, None, type) | ||
|
r43347 | fp = httpconnection.httpsendfile(self.ui, tempname, b"rb") | ||
|
r36311 | headers = {r'Content-Type': r'application/mercurial-0.1'} | ||
|
r17192 | |||
try: | ||||
|
r25085 | r = self._call(cmd, data=fp, headers=headers, **args) | ||
|
r43347 | vals = r.split(b'\n', 1) | ||
|
r25085 | if len(vals) < 2: | ||
|
r43347 | raise error.ResponseError(_(b"unexpected response:"), r) | ||
|
r25085 | return vals | ||
|
r36448 | except urlerr.httperror: | ||
# Catch and re-raise these so we don't try and treat them | ||||
# like generic socket errors. They lack any values in | ||||
# .args on Python 3 which breaks our socket.error block. | ||||
raise | ||||
|
r25660 | except socket.error as err: | ||
|
r25085 | if err.args[0] in (errno.ECONNRESET, errno.EPIPE): | ||
|
r43347 | raise error.Abort(_(b'push failed: %s') % err.args[1]) | ||
|
r26587 | raise error.Abort(err.args[1]) | ||
|
r17192 | finally: | ||
fp.close() | ||||
os.unlink(tempname) | ||||
|
r21074 | def _calltwowaystream(self, cmd, fp, **args): | ||
filename = None | ||||
try: | ||||
# dump bundle to disk | ||||
|
r43347 | fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg") | ||
|
r43118 | with os.fdopen(fd, r"wb") as fh: | ||
|
r21074 | d = fp.read(4096) | ||
|
r43118 | while d: | ||
fh.write(d) | ||||
d = fp.read(4096) | ||||
|
r21074 | # start http push | ||
|
r43347 | with httpconnection.httpsendfile(self.ui, filename, b"rb") as fp_: | ||
|
r43119 | headers = {r'Content-Type': r'application/mercurial-0.1'} | ||
return self._callstream(cmd, data=fp_, headers=headers, **args) | ||||
|
r21074 | finally: | ||
|
r43118 | if filename is not None: | ||
|
r21074 | os.unlink(filename) | ||
|
r20905 | def _callcompressable(self, cmd, **args): | ||
|
r30464 | return self._callstream(cmd, _compressible=True, **args) | ||
|
r17192 | |||
|
r21188 | def _abort(self, exception): | ||
raise exception | ||||
|
r43346 | |||
def sendv2request( | ||||
ui, opener, requestbuilder, apiurl, permission, requests, redirect | ||||
): | ||||
|
r40168 | wireprotoframing.populatestreamencoders() | ||
uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order') | ||||
if uiencoders: | ||||
encoders = [] | ||||
for encoder in uiencoders: | ||||
if encoder not in wireprotoframing.STREAM_ENCODERS: | ||||
|
r43346 | ui.warn( | ||
_( | ||||
b'wire protocol version 2 encoder referenced in ' | ||||
b'config (%s) is not known; ignoring\n' | ||||
) | ||||
% encoder | ||||
) | ||||
|
r40168 | else: | ||
encoders.append(encoder) | ||||
else: | ||||
encoders = wireprotoframing.STREAM_ENCODERS_ORDER | ||||
|
r43346 | reactor = wireprotoframing.clientreactor( | ||
ui, | ||||
hasmultiplesend=False, | ||||
buffersends=True, | ||||
clientcontentencoders=encoders, | ||||
) | ||||
|
r37669 | |||
|
r43346 | handler = wireprotov2peer.clienthandler( | ||
ui, reactor, opener=opener, requestbuilder=requestbuilder | ||||
) | ||||
|
r37737 | |||
|
r43347 | url = b'%s/%s' % (apiurl, permission) | ||
|
r37669 | |||
if len(requests) > 1: | ||||
|
r43347 | url += b'/multirequest' | ||
|
r37669 | else: | ||
|
r43347 | url += b'/%s' % requests[0][0] | ||
|
r37669 | |||
|
r43347 | ui.debug(b'sending %d commands\n' % len(requests)) | ||
|
r37669 | for command, args, f in requests: | ||
|
r43346 | ui.debug( | ||
|
r43347 | b'sending command %s: %s\n' | ||
|
r43346 | % (command, stringutil.pprint(args, indent=2)) | ||
) | ||||
assert not list( | ||||
handler.callcommand(command, args, f, redirect=redirect) | ||||
) | ||||
|
r37669 | |||
# TODO stream this. | ||||
|
r37737 | body = b''.join(map(bytes, handler.flushcommands())) | ||
|
r37669 | |||
# TODO modify user-agent to reflect v2 | ||||
headers = { | ||||
r'Accept': wireprotov2server.FRAMINGTYPE, | ||||
r'Content-Type': wireprotov2server.FRAMINGTYPE, | ||||
} | ||||
req = requestbuilder(pycompat.strurl(url), body, headers) | ||||
req.add_unredirected_header(r'Content-Length', r'%d' % len(body)) | ||||
try: | ||||
res = opener.open(req) | ||||
except urlerr.httperror as e: | ||||
if e.code == 401: | ||||
|
r43347 | raise error.Abort(_(b'authorization failed')) | ||
|
r37669 | |||
raise | ||||
except httplib.HTTPException as e: | ||||
ui.traceback() | ||||
raise IOError(None, e) | ||||
|
r37737 | return handler, res | ||
|
r37669 | |||
|
r43346 | |||
|
r37669 | class queuedcommandfuture(pycompat.futures.Future): | ||
"""Wraps result() on command futures to trigger submission on call.""" | ||||
def result(self, timeout=None): | ||||
if self.done(): | ||||
return pycompat.futures.Future.result(self, timeout) | ||||
self._peerexecutor.sendcommands() | ||||
# sendcommands() will restore the original __class__ and self.result | ||||
# will resolve to Future.result. | ||||
return self.result(timeout) | ||||
|
r43346 | |||
|
r37828 | @interfaceutil.implementer(repository.ipeercommandexecutor) | ||
|
r37669 | class httpv2executor(object): | ||
|
r43346 | def __init__( | ||
self, ui, opener, requestbuilder, apiurl, descriptor, redirect | ||||
): | ||||
|
r37669 | self._ui = ui | ||
self._opener = opener | ||||
self._requestbuilder = requestbuilder | ||||
self._apiurl = apiurl | ||||
self._descriptor = descriptor | ||||
|
r40060 | self._redirect = redirect | ||
|
r37669 | self._sent = False | ||
self._closed = False | ||||
self._neededpermissions = set() | ||||
self._calls = [] | ||||
self._futures = weakref.WeakSet() | ||||
self._responseexecutor = None | ||||
self._responsef = None | ||||
def __enter__(self): | ||||
return self | ||||
def __exit__(self, exctype, excvalue, exctb): | ||||
self.close() | ||||
def callcommand(self, command, args): | ||||
if self._sent: | ||||
|
r43346 | raise error.ProgrammingError( | ||
|
r43387 | b'callcommand() cannot be used after commands are sent' | ||
|
r43346 | ) | ||
|
r37669 | |||
if self._closed: | ||||
|
r43346 | raise error.ProgrammingError( | ||
|
r43387 | b'callcommand() cannot be used after close()' | ||
|
r43346 | ) | ||
|
r37669 | |||
# The service advertises which commands are available. So if we attempt | ||||
# to call an unknown command or pass an unknown argument, we can screen | ||||
# for this. | ||||
|
r43347 | if command not in self._descriptor[b'commands']: | ||
|
r37669 | raise error.ProgrammingError( | ||
|
r43347 | b'wire protocol command %s is not available' % command | ||
|
r43346 | ) | ||
|
r37669 | |||
|
r43347 | cmdinfo = self._descriptor[b'commands'][command] | ||
unknownargs = set(args.keys()) - set(cmdinfo.get(b'args', {})) | ||||
|
r37669 | |||
if unknownargs: | ||||
raise error.ProgrammingError( | ||||
|
r43347 | b'wire protocol command %s does not accept argument: %s' | ||
% (command, b', '.join(sorted(unknownargs))) | ||||
|
r43346 | ) | ||
|
r37669 | |||
|
r43347 | self._neededpermissions |= set(cmdinfo[b'permissions']) | ||
|
r37669 | |||
# TODO we /could/ also validate types here, since the API descriptor | ||||
# includes types... | ||||
f = pycompat.futures.Future() | ||||
# Monkeypatch it so result() triggers sendcommands(), otherwise result() | ||||
# could deadlock. | ||||
f.__class__ = queuedcommandfuture | ||||
f._peerexecutor = self | ||||
self._futures.add(f) | ||||
self._calls.append((command, args, f)) | ||||
return f | ||||
def sendcommands(self): | ||||
if self._sent: | ||||
return | ||||
if not self._calls: | ||||
return | ||||
self._sent = True | ||||
# Unhack any future types so caller sees a clean type and so we | ||||
# break reference cycle. | ||||
for f in self._futures: | ||||
if isinstance(f, queuedcommandfuture): | ||||
f.__class__ = pycompat.futures.Future | ||||
f._peerexecutor = None | ||||
# Mark the future as running and filter out cancelled futures. | ||||
|
r43346 | calls = [ | ||
(command, args, f) | ||||
for command, args, f in self._calls | ||||
if f.set_running_or_notify_cancel() | ||||
] | ||||
|
r37669 | |||
# Clear out references, prevent improper object usage. | ||||
self._calls = None | ||||
if not calls: | ||||
return | ||||
permissions = set(self._neededpermissions) | ||||
|
r43347 | if b'push' in permissions and b'pull' in permissions: | ||
permissions.remove(b'pull') | ||||
|
r37669 | |||
if len(permissions) > 1: | ||||
|
r43346 | raise error.RepoError( | ||
|
r43387 | _(b'cannot make request requiring multiple permissions: %s') | ||
|
r43347 | % _(b', ').join(sorted(permissions)) | ||
|
r43346 | ) | ||
|
r37669 | |||
|
r43347 | permission = {b'push': b'rw', b'pull': b'ro',}[permissions.pop()] | ||
|
r37669 | |||
|
r37737 | handler, resp = sendv2request( | ||
|
r43346 | self._ui, | ||
self._opener, | ||||
self._requestbuilder, | ||||
self._apiurl, | ||||
permission, | ||||
calls, | ||||
self._redirect, | ||||
) | ||||
|
r37669 | |||
# TODO we probably want to validate the HTTP code, media type, etc. | ||||
self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1) | ||||
|
r43346 | self._responsef = self._responseexecutor.submit( | ||
self._handleresponse, handler, resp | ||||
) | ||||
|
r37669 | |||
def close(self): | ||||
if self._closed: | ||||
return | ||||
self.sendcommands() | ||||
self._closed = True | ||||
if not self._responsef: | ||||
return | ||||
|
r39468 | # TODO ^C here may not result in immediate program termination. | ||
|
r37669 | try: | ||
self._responsef.result() | ||||
finally: | ||||
self._responseexecutor.shutdown(wait=True) | ||||
self._responsef = None | ||||
self._responseexecutor = None | ||||
# If any of our futures are still in progress, mark them as | ||||
# errored, otherwise a result() could wait indefinitely. | ||||
for f in self._futures: | ||||
if not f.done(): | ||||
|
r43346 | f.set_exception( | ||
|
r43347 | error.ResponseError(_(b'unfulfilled command response')) | ||
|
r43346 | ) | ||
|
r37669 | |||
self._futures = None | ||||
|
r37737 | def _handleresponse(self, handler, resp): | ||
|
r37669 | # Called in a thread to read the response. | ||
|
r40055 | while handler.readdata(resp): | ||
|
r37737 | pass | ||
|
r37669 | |||
|
r43346 | |||
|
r40207 | @interfaceutil.implementer(repository.ipeerv2) | ||
|
r37501 | class httpv2peer(object): | ||
r42334 | ||||
limitedarguments = False | ||||
|
r43346 | def __init__( | ||
self, ui, repourl, apipath, opener, requestbuilder, apidescriptor | ||||
): | ||||
|
r37501 | self.ui = ui | ||
|
r40207 | self.apidescriptor = apidescriptor | ||
|
r37501 | |||
|
r43347 | if repourl.endswith(b'/'): | ||
|
r37501 | repourl = repourl[:-1] | ||
|
r37627 | self._url = repourl | ||
|
r37576 | self._apipath = apipath | ||
|
r43347 | self._apiurl = b'%s/%s' % (repourl, apipath) | ||
|
r37501 | self._opener = opener | ||
|
r37576 | self._requestbuilder = requestbuilder | ||
|
r37501 | |||
|
r40060 | self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor) | ||
|
r37627 | # Start of ipeerconnection. | ||
def url(self): | ||||
return self._url | ||||
def local(self): | ||||
return None | ||||
def peer(self): | ||||
return self | ||||
def canpush(self): | ||||
# TODO change once implemented. | ||||
return False | ||||
|
r37501 | def close(self): | ||
|
r43346 | self.ui.note( | ||
_( | ||||
|
r43347 | b'(sent %d HTTP requests and %d bytes; ' | ||
b'received %d bytes in responses)\n' | ||||
|
r43346 | ) | ||
% ( | ||||
self._opener.requestscount, | ||||
self._opener.sentbytescount, | ||||
self._opener.receivedbytescount, | ||||
) | ||||
) | ||||
|
r37501 | |||
|
r37627 | # End of ipeerconnection. | ||
|
r37629 | # Start of ipeercapabilities. | ||
def capable(self, name): | ||||
# The capabilities used internally historically map to capabilities | ||||
# advertised from the "capabilities" wire protocol command. However, | ||||
# version 2 of that command works differently. | ||||
# Maps to commands that are available. | ||||
|
r43347 | if name in ( | ||
b'branchmap', | ||||
b'getbundle', | ||||
b'known', | ||||
b'lookup', | ||||
b'pushkey', | ||||
): | ||||
|
r37629 | return True | ||
# Other concepts. | ||||
|
r43347 | if name in b'bundle2': | ||
|
r37629 | return True | ||
|
r39664 | # Alias command-* to presence of command of that name. | ||
|
r43347 | if name.startswith(b'command-'): | ||
return name[len(b'command-') :] in self.apidescriptor[b'commands'] | ||||
|
r39664 | |||
|
r37629 | return False | ||
def requirecap(self, name, purpose): | ||||
if self.capable(name): | ||||
return | ||||
raise error.CapabilityError( | ||||
|
r43346 | _( | ||
|
r43347 | b'cannot %s; client or remote repository does not support the ' | ||
b'\'%s\' capability' | ||||
|
r43346 | ) | ||
% (purpose, name) | ||||
) | ||||
|
r37629 | |||
# End of ipeercapabilities. | ||||
|
r37501 | def _call(self, name, **args): | ||
|
r37669 | with self.commandexecutor() as e: | ||
return e.callcommand(name, args).result() | ||||
|
r37561 | |||
|
r37669 | def commandexecutor(self): | ||
|
r43346 | return httpv2executor( | ||
self.ui, | ||||
self._opener, | ||||
self._requestbuilder, | ||||
self._apiurl, | ||||
self.apidescriptor, | ||||
self._redirect, | ||||
) | ||||
|
r37501 | |||
|
r37576 | # Registry of API service names to metadata about peers that handle it. | ||
# | ||||
# The following keys are meaningful: | ||||
# | ||||
# init | ||||
# Callable receiving (ui, repourl, servicepath, opener, requestbuilder, | ||||
# apidescriptor) to create a peer. | ||||
# | ||||
# priority | ||||
# Integer priority for the service. If we could choose from multiple | ||||
# services, we choose the one with the highest priority. | ||||
API_PEERS = { | ||||
|
r43347 | wireprototypes.HTTP_WIREPROTO_V2: {b'init': httpv2peer, b'priority': 50,}, | ||
|
r37576 | } | ||
|
r43346 | |||
|
r37570 | def performhandshake(ui, url, opener, requestbuilder): | ||
# The handshake is a request to the capabilities command. | ||||
caps = None | ||||
|
r43346 | |||
|
r37570 | def capable(x): | ||
|
r43347 | raise error.ProgrammingError(b'should not be called') | ||
|
r37570 | |||
|
r37576 | args = {} | ||
# The client advertises support for newer protocols by adding an | ||||
# X-HgUpgrade-* header with a list of supported APIs and an | ||||
# X-HgProto-* header advertising which serializing formats it supports. | ||||
# We only support the HTTP version 2 transport and CBOR responses for | ||||
# now. | ||||
|
r43347 | advertisev2 = ui.configbool(b'experimental', b'httppeer.advertise-v2') | ||
|
r37576 | |||
if advertisev2: | ||||
|
r43347 | args[b'headers'] = { | ||
|
r37576 | r'X-HgProto-1': r'cbor', | ||
} | ||||
|
r43347 | args[b'headers'].update( | ||
|
r43346 | encodevalueinheaders( | ||
|
r43347 | b' '.join(sorted(API_PEERS)), | ||
b'X-HgUpgrade', | ||||
|
r43346 | # We don't know the header limit this early. | ||
# So make it small. | ||||
1024, | ||||
) | ||||
) | ||||
|
r37576 | |||
|
r43346 | req, requrl, qs = makev1commandrequest( | ||
|
r43347 | ui, requestbuilder, caps, capable, url, b'capabilities', args | ||
|
r43346 | ) | ||
|
r37570 | resp = sendrequest(ui, opener, req) | ||
|
r37851 | # The server may redirect us to the repo root, stripping the | ||
# ?cmd=capabilities query string from the URL. The server would likely | ||||
# return HTML in this case and ``parsev1commandresponse()`` would raise. | ||||
# We catch this special case and re-issue the capabilities request against | ||||
# the new URL. | ||||
# | ||||
# We should ideally not do this, as a redirect that drops the query | ||||
# string from the URL is arguably a server bug. (Garbage in, garbage out). | ||||
# However, Mercurial clients for several years appeared to handle this | ||||
# issue without behavior degradation. And according to issue 5860, it may | ||||
# be a longstanding bug in some server implementations. So we allow a | ||||
# redirect that drops the query string to "just work." | ||||
try: | ||||
|
r43346 | respurl, ct, resp = parsev1commandresponse( | ||
ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2 | ||||
) | ||||
|
r37851 | except RedirectedRepoError as e: | ||
|
r43346 | req, requrl, qs = makev1commandrequest( | ||
|
r43347 | ui, requestbuilder, caps, capable, e.respurl, b'capabilities', args | ||
|
r43346 | ) | ||
|
r37851 | resp = sendrequest(ui, opener, req) | ||
|
r43346 | respurl, ct, resp = parsev1commandresponse( | ||
ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2 | ||||
) | ||||
|
r37570 | |||
try: | ||||
|
r37576 | rawdata = resp.read() | ||
|
r37570 | finally: | ||
resp.close() | ||||
|
r43347 | if not ct.startswith(b'application/mercurial-'): | ||
raise error.ProgrammingError(b'unexpected content-type: %s' % ct) | ||||
|
r37576 | |||
if advertisev2: | ||||
|
r43347 | if ct == b'application/mercurial-cbor': | ||
|
r37576 | try: | ||
|
r39476 | info = cborutil.decodeall(rawdata)[0] | ||
except cborutil.CBORDecodeError: | ||||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'error decoding CBOR from remote server'), | ||
|
r43346 | hint=_( | ||
|
r43347 | b'try again and consider contacting ' | ||
b'the server operator' | ||||
|
r43346 | ), | ||
) | ||||
|
r37576 | |||
# We got a legacy response. That's fine. | ||||
|
r43347 | elif ct in (b'application/mercurial-0.1', b'application/mercurial-0.2'): | ||
info = {b'v1capabilities': set(rawdata.split())} | ||||
|
r37576 | |||
else: | ||||
raise error.RepoError( | ||||
|
r43347 | _(b'unexpected response type from server: %s') % ct | ||
|
r43346 | ) | ||
|
r37576 | else: | ||
|
r43347 | info = {b'v1capabilities': set(rawdata.split())} | ||
|
r37576 | |||
return respurl, info | ||||
|
r37570 | |||
|
r43346 | |||
|
r37571 | def makepeer(ui, path, opener=None, requestbuilder=urlreq.request): | ||
|
r37565 | """Construct an appropriate HTTP peer instance. | ||
|
r37571 | ``opener`` is an ``url.opener`` that should be used to establish | ||
connections, perform HTTP requests. | ||||
|
r37565 | ``requestbuilder`` is the type used for constructing HTTP requests. | ||
It exists as an argument so extensions can override the default. | ||||
""" | ||||
|
r37024 | u = util.url(path) | ||
if u.query or u.fragment: | ||||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'unsupported URL component: "%s"') % (u.query or u.fragment) | ||
|
r43346 | ) | ||
|
r37024 | |||
# urllib cannot handle URLs with embedded user or passwd. | ||||
url, authinfo = u.authinfo() | ||||
|
r43347 | ui.debug(b'using %s\n' % url) | ||
|
r37024 | |||
|
r37571 | opener = opener or urlmod.opener(ui, authinfo) | ||
|
r37024 | |||
|
r37576 | respurl, info = performhandshake(ui, url, opener, requestbuilder) | ||
# Given the intersection of APIs that both we and the server support, | ||||
# sort by their advertised priority and pick the first one. | ||||
# | ||||
# TODO consider making this request-based and interface driven. For | ||||
# example, the caller could say "I want a peer that does X." It's quite | ||||
# possible that not all peers would do that. Since we know the service | ||||
# capabilities, we could filter out services not meeting the | ||||
# requirements. Possibly by consulting the interfaces defined by the | ||||
# peer type. | ||||
|
r43347 | apipeerchoices = set(info.get(b'apis', {}).keys()) & set(API_PEERS.keys()) | ||
|
r37570 | |||
|
r43346 | preferredchoices = sorted( | ||
|
r43347 | apipeerchoices, key=lambda x: API_PEERS[x][b'priority'], reverse=True | ||
|
r43346 | ) | ||
|
r37576 | |||
for service in preferredchoices: | ||||
|
r43347 | apipath = b'%s/%s' % (info[b'apibase'].rstrip(b'/'), service) | ||
|
r37576 | |||
|
r43347 | return API_PEERS[service][b'init']( | ||
ui, respurl, apipath, opener, requestbuilder, info[b'apis'][service] | ||||
|
r43346 | ) | ||
|
r37576 | |||
# Failed to construct an API peer. Fall back to legacy. | ||||
|
r43346 | return httppeer( | ||
|
r43347 | ui, path, respurl, opener, requestbuilder, info[b'v1capabilities'] | ||
|
r43346 | ) | ||
|
r37024 | |||
|
r39585 | def instance(ui, path, create, intents=None, createopts=None): | ||
|
r17192 | if create: | ||
|
r43347 | raise error.Abort(_(b'cannot create new http repository')) | ||
|
r17192 | try: | ||
|
r43347 | if path.startswith(b'https:') and not urlmod.has_https: | ||
|
r43346 | raise error.Abort( | ||
|
r43387 | _(b'Python support for SSL and HTTPS is not installed') | ||
|
r43346 | ) | ||
|
r35902 | |||
|
r37024 | inst = makepeer(ui, path) | ||
|
r35902 | |||
|
r17192 | return inst | ||
|
r25660 | except error.RepoError as httpexception: | ||
|
r17192 | try: | ||
|
r43347 | r = statichttprepo.instance(ui, b"static-" + path, create) | ||
ui.note(_(b'(falling back to static-http)\n')) | ||||
|
r17192 | return r | ||
except error.RepoError: | ||||
|
r43346 | raise httpexception # use the original http RepoError instead | ||