httppeer.py
464 lines
| 16.2 KiB
| text/x-python
|
PythonLexer
/ mercurial / httppeer.py
Peter Arrenbrecht
|
r17192 | # httppeer.py - HTTP repository proxy classes for mercurial | ||
# | ||||
# Copyright 2005, 2006 Matt Mackall <mpm@selenic.com> | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
Gregory Szorc
|
r25954 | from __future__ import absolute_import | ||
import errno | ||||
Augie Fackler
|
r33820 | import io | ||
Gregory Szorc
|
r25954 | import os | ||
import socket | ||||
Gregory Szorc
|
r30763 | import struct | ||
Pierre-Yves David
|
r21074 | import tempfile | ||
Gregory Szorc
|
r25954 | |||
from .i18n import _ | ||||
from .node import nullid | ||||
from . import ( | ||||
Martin von Zweigbergk
|
r28666 | bundle2, | ||
Gregory Szorc
|
r25954 | error, | ||
httpconnection, | ||||
Pulkit Goyal
|
r30924 | pycompat, | ||
Gregory Szorc
|
r25954 | statichttprepo, | ||
url, | ||||
util, | ||||
wireproto, | ||||
) | ||||
Peter Arrenbrecht
|
r17192 | |||
Pulkit Goyal
|
r29455 | httplib = util.httplib | ||
timeless
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
Gregory Szorc
|
r30759 | def encodevalueinheaders(value, header, limit): | ||
"""Encode a string value into multiple HTTP headers. | ||||
``value`` will be encoded into 1 or more HTTP headers with the names | ||||
``header-<N>`` where ``<N>`` is an integer starting at 1. Each header | ||||
name + value will be at most ``limit`` bytes long. | ||||
Returns an iterable of 2-tuples consisting of header names and values. | ||||
""" | ||||
fmt = header + '-%s' | ||||
valuelen = limit - len(fmt % '000') - len(': \r\n') | ||||
result = [] | ||||
n = 0 | ||||
for i in xrange(0, len(value), valuelen): | ||||
n += 1 | ||||
result.append((fmt % str(n), value[i:i + valuelen])) | ||||
return result | ||||
Gregory Szorc
|
r32002 | def _wraphttpresponse(resp): | ||
"""Wrap an HTTPResponse with common error handlers. | ||||
This ensures that any I/O from any consumer raises the appropriate | ||||
error and messaging. | ||||
""" | ||||
origread = resp.read | ||||
class readerproxy(resp.__class__): | ||||
def read(self, size=None): | ||||
try: | ||||
return origread(size) | ||||
except httplib.IncompleteRead as e: | ||||
# e.expected is an integer if length known or None otherwise. | ||||
if e.expected: | ||||
msg = _('HTTP request error (incomplete response; ' | ||||
'expected %d bytes got %d)') % (e.expected, | ||||
len(e.partial)) | ||||
else: | ||||
msg = _('HTTP request error (incomplete response)') | ||||
Gregory Szorc
|
r32023 | raise error.PeerTransportError( | ||
Gregory Szorc
|
r32002 | msg, | ||
hint=_('this may be an intermittent network failure; ' | ||||
'if the error persists, consider contacting the ' | ||||
'network or server operator')) | ||||
except httplib.HTTPException as e: | ||||
Gregory Szorc
|
r32023 | raise error.PeerTransportError( | ||
Gregory Szorc
|
r32002 | _('HTTP request error (%s)') % e, | ||
FUJIWARA Katsunori
|
r32087 | hint=_('this may be an intermittent network failure; ' | ||
Gregory Szorc
|
r32002 | 'if the error persists, consider contacting the ' | ||
'network or server operator')) | ||||
resp.__class__ = readerproxy | ||||
Augie Fackler
|
r33820 | class _multifile(object): | ||
def __init__(self, *fileobjs): | ||||
for f in fileobjs: | ||||
if not util.safehasattr(f, 'length'): | ||||
raise ValueError( | ||||
'_multifile only supports file objects that ' | ||||
'have a length but this one does not:', type(f), f) | ||||
self._fileobjs = fileobjs | ||||
self._index = 0 | ||||
@property | ||||
def length(self): | ||||
return sum(f.length for f in self._fileobjs) | ||||
def read(self, amt=None): | ||||
if amt <= 0: | ||||
return ''.join(f.read() for f in self._fileobjs) | ||||
parts = [] | ||||
while amt and self._index < len(self._fileobjs): | ||||
parts.append(self._fileobjs[self._index].read(amt)) | ||||
got = len(parts[-1]) | ||||
if got < amt: | ||||
self._index += 1 | ||||
amt -= got | ||||
return ''.join(parts) | ||||
def seek(self, offset, whence=os.SEEK_SET): | ||||
if whence != os.SEEK_SET: | ||||
raise NotImplementedError( | ||||
'_multifile does not support anything other' | ||||
' than os.SEEK_SET for whence on seek()') | ||||
if offset != 0: | ||||
raise NotImplementedError( | ||||
'_multifile only supports seeking to start, but that ' | ||||
'could be fixed if you need it') | ||||
for f in self._fileobjs: | ||||
f.seek(0) | ||||
self._index = 0 | ||||
Gregory Szorc
|
r33805 | class httppeer(wireproto.wirepeer): | ||
Peter Arrenbrecht
|
r17192 | def __init__(self, ui, path): | ||
Gregory Szorc
|
r33671 | self._path = path | ||
self._caps = None | ||||
self._urlopener = None | ||||
self._requestbuilder = None | ||||
Peter Arrenbrecht
|
r17192 | u = util.url(path) | ||
if u.query or u.fragment: | ||||
Pierre-Yves David
|
r26587 | raise error.Abort(_('unsupported URL component: "%s"') % | ||
Peter Arrenbrecht
|
r17192 | (u.query or u.fragment)) | ||
# urllib cannot handle URLs with embedded user or passwd | ||||
self._url, authinfo = u.authinfo() | ||||
Gregory Szorc
|
r33804 | self._ui = ui | ||
ui.debug('using %s\n' % self._url) | ||||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33671 | self._urlopener = url.opener(ui, authinfo) | ||
self._requestbuilder = urlreq.request | ||||
Peter Arrenbrecht
|
r17192 | |||
def __del__(self): | ||||
Gregory Szorc
|
r33671 | urlopener = getattr(self, '_urlopener', None) | ||
Mads Kiilerich
|
r30241 | if urlopener: | ||
for h in urlopener.handlers: | ||||
Peter Arrenbrecht
|
r17192 | h.close() | ||
Alex Gaynor
|
r34487 | getattr(h, "close_all", lambda: None)() | ||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33804 | # Begin of _basepeer interface. | ||
@util.propertycache | ||||
def ui(self): | ||||
return self._ui | ||||
Peter Arrenbrecht
|
r17192 | def url(self): | ||
Gregory Szorc
|
r33671 | return self._path | ||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33804 | def local(self): | ||
return None | ||||
def peer(self): | ||||
return self | ||||
def canpush(self): | ||||
return True | ||||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33804 | def close(self): | ||
pass | ||||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33804 | # End of _basepeer interface. | ||
# Begin of _basewirepeer interface. | ||||
def capabilities(self): | ||||
Gregory Szorc
|
r33671 | if self._caps is None: | ||
Peter Arrenbrecht
|
r17192 | try: | ||
self._fetchcaps() | ||||
except error.RepoError: | ||||
Gregory Szorc
|
r33671 | self._caps = set() | ||
Peter Arrenbrecht
|
r17192 | self.ui.debug('capabilities: %s\n' % | ||
Gregory Szorc
|
r33671 | (' '.join(self._caps or ['none']))) | ||
return self._caps | ||||
Peter Arrenbrecht
|
r17192 | |||
Gregory Szorc
|
r33804 | # End of _basewirepeer interface. | ||
# look up capabilities only when needed | ||||
def _fetchcaps(self): | ||||
self._caps = set(self._call('capabilities').split()) | ||||
Gregory Szorc
|
r30464 | def _callstream(self, cmd, _compressible=False, **args): | ||
Peter Arrenbrecht
|
r17192 | if cmd == 'pushkey': | ||
args['data'] = '' | ||||
data = args.pop('data', None) | ||||
headers = args.pop('headers', {}) | ||||
self.ui.debug("sending %s command\n" % cmd) | ||||
q = [('cmd', cmd)] | ||||
headersize = 0 | ||||
Gregory Szorc
|
r30564 | varyheaders = [] | ||
Augie Fackler
|
r28530 | # Important: don't use self.capable() here or else you end up | ||
# with infinite recursion when trying to look up capabilities | ||||
# for the first time. | ||||
Gregory Szorc
|
r33671 | postargsok = self._caps is not None and 'httppostargs' in self._caps | ||
Augie Fackler
|
r33820 | if postargsok and args: | ||
timeless
|
r28883 | strargs = urlreq.urlencode(sorted(args.items())) | ||
Augie Fackler
|
r33820 | if not data: | ||
data = strargs | ||||
else: | ||||
if isinstance(data, basestring): | ||||
i = io.BytesIO(data) | ||||
i.length = len(data) | ||||
data = i | ||||
argsio = io.BytesIO(strargs) | ||||
argsio.length = len(strargs) | ||||
data = _multifile(argsio, data) | ||||
Augie Fackler
|
r34702 | headers[r'X-HgArgs-Post'] = len(strargs) | ||
Augie Fackler
|
r28530 | else: | ||
Augie Fackler
|
r28485 | if len(args) > 0: | ||
httpheader = self.capable('httpheader') | ||||
if httpheader: | ||||
headersize = int(httpheader.split(',', 1)[0]) | ||||
if headersize > 0: | ||||
# The headers can typically carry more data than the URL. | ||||
timeless
|
r28883 | encargs = urlreq.urlencode(sorted(args.items())) | ||
Gregory Szorc
|
r30759 | for header, value in encodevalueinheaders(encargs, 'X-HgArg', | ||
headersize): | ||||
headers[header] = value | ||||
Augie Fackler
|
r28486 | varyheaders.append(header) | ||
Augie Fackler
|
r28485 | else: | ||
q += sorted(args.items()) | ||||
timeless
|
r28883 | qs = '?%s' % urlreq.urlencode(q) | ||
Peter Arrenbrecht
|
r17192 | cu = "%s%s" % (self._url, qs) | ||
Augie Fackler
|
r28484 | size = 0 | ||
if util.safehasattr(data, 'length'): | ||||
size = data.length | ||||
elif data is not None: | ||||
size = len(data) | ||||
Jun Wu
|
r33499 | if size and self.ui.configbool('ui', 'usehttp2'): | ||
Augie Fackler
|
r34702 | headers[r'Expect'] = r'100-Continue' | ||
headers[r'X-HgHttp2'] = r'1' | ||||
if data is not None and r'Content-Type' not in headers: | ||||
headers[r'Content-Type'] = r'application/mercurial-0.1' | ||||
Gregory Szorc
|
r30564 | |||
Gregory Szorc
|
r30763 | # Tell the server we accept application/mercurial-0.2 and multiple | ||
# compression formats if the server is capable of emitting those | ||||
# payloads. | ||||
protoparams = [] | ||||
mediatypes = set() | ||||
Gregory Szorc
|
r33671 | if self._caps is not None: | ||
Gregory Szorc
|
r30763 | mt = self.capable('httpmediatype') | ||
if mt: | ||||
protoparams.append('0.1') | ||||
mediatypes = set(mt.split(',')) | ||||
if '0.2tx' in mediatypes: | ||||
protoparams.append('0.2') | ||||
if '0.2tx' in mediatypes and self.capable('compression'): | ||||
# We /could/ compare supported compression formats and prune | ||||
# non-mutually supported or error if nothing is mutually supported. | ||||
# For now, send the full list to the server and have it error. | ||||
comps = [e.wireprotosupport().name for e in | ||||
util.compengines.supportedwireengines(util.CLIENTROLE)] | ||||
protoparams.append('comp=%s' % ','.join(comps)) | ||||
if protoparams: | ||||
protoheaders = encodevalueinheaders(' '.join(protoparams), | ||||
'X-HgProto', | ||||
headersize or 1024) | ||||
for header, value in protoheaders: | ||||
headers[header] = value | ||||
varyheaders.append(header) | ||||
Gregory Szorc
|
r32022 | if varyheaders: | ||
Augie Fackler
|
r34702 | headers[r'Vary'] = r','.join(varyheaders) | ||
Gregory Szorc
|
r32022 | |||
Augie Fackler
|
r34700 | req = self._requestbuilder(pycompat.strurl(cu), data, headers) | ||
Gregory Szorc
|
r30564 | |||
Peter Arrenbrecht
|
r17192 | if data is not None: | ||
self.ui.debug("sending %s bytes\n" % size) | ||||
req.add_unredirected_header('Content-Length', '%d' % size) | ||||
try: | ||||
Gregory Szorc
|
r33671 | resp = self._urlopener.open(req) | ||
timeless
|
r28883 | except urlerr.httperror as inst: | ||
Peter Arrenbrecht
|
r17192 | if inst.code == 401: | ||
Pierre-Yves David
|
r26587 | raise error.Abort(_('authorization failed')) | ||
Peter Arrenbrecht
|
r17192 | raise | ||
Gregory Szorc
|
r25660 | except httplib.HTTPException as inst: | ||
Peter Arrenbrecht
|
r17192 | self.ui.debug('http error while sending %s command\n' % cmd) | ||
self.ui.traceback() | ||||
raise IOError(None, inst) | ||||
Gregory Szorc
|
r32002 | |||
# Insert error handlers for common I/O failures. | ||||
_wraphttpresponse(resp) | ||||
Peter Arrenbrecht
|
r17192 | # record the url we got redirected to | ||
resp_url = resp.geturl() | ||||
if resp_url.endswith(qs): | ||||
resp_url = resp_url[:-len(qs)] | ||||
if self._url.rstrip('/') != resp_url.rstrip('/'): | ||||
if not self.ui.quiet: | ||||
self.ui.warn(_('real URL is %s\n') % resp_url) | ||||
self._url = resp_url | ||||
try: | ||||
proto = resp.getheader('content-type') | ||||
except AttributeError: | ||||
proto = resp.headers.get('content-type', '') | ||||
safeurl = util.hidepassword(self._url) | ||||
if proto.startswith('application/hg-error'): | ||||
raise error.OutOfBandError(resp.read()) | ||||
# accept old "text/plain" and "application/hg-changegroup" for now | ||||
if not (proto.startswith('application/mercurial-') or | ||||
Matt Mackall
|
r18737 | (proto.startswith('text/plain') | ||
and not resp.headers.get('content-length')) or | ||||
Peter Arrenbrecht
|
r17192 | proto.startswith('application/hg-changegroup')): | ||
self.ui.debug("requested URL: '%s'\n" % util.hidepassword(cu)) | ||||
raise error.RepoError( | ||||
_("'%s' does not appear to be an hg repository:\n" | ||||
"---%%<--- (%s)\n%s\n---%%<---\n") | ||||
Matt Mackall
|
r18738 | % (safeurl, proto or 'no content-type', resp.read(1024))) | ||
Peter Arrenbrecht
|
r17192 | |||
if proto.startswith('application/mercurial-'): | ||||
try: | ||||
version = proto.split('-', 1)[1] | ||||
version_info = tuple([int(n) for n in version.split('.')]) | ||||
except ValueError: | ||||
raise error.RepoError(_("'%s' sent a broken Content-Type " | ||||
"header (%s)") % (safeurl, proto)) | ||||
Gregory Szorc
|
r30763 | |||
Gregory Szorc
|
r32003 | # TODO consider switching to a decompression reader that uses | ||
# generators. | ||||
Gregory Szorc
|
r30763 | if version_info == (0, 1): | ||
if _compressible: | ||||
Gregory Szorc
|
r32003 | return util.compengines['zlib'].decompressorreader(resp) | ||
Gregory Szorc
|
r30763 | return resp | ||
elif version_info == (0, 2): | ||||
# application/mercurial-0.2 always identifies the compression | ||||
# engine in the payload header. | ||||
elen = struct.unpack('B', resp.read(1))[0] | ||||
ename = resp.read(elen) | ||||
engine = util.compengines.forwiretype(ename) | ||||
Gregory Szorc
|
r32003 | return engine.decompressorreader(resp) | ||
Gregory Szorc
|
r30763 | else: | ||
Peter Arrenbrecht
|
r17192 | raise error.RepoError(_("'%s' uses newer protocol %s") % | ||
(safeurl, version)) | ||||
Gregory Szorc
|
r30464 | if _compressible: | ||
Gregory Szorc
|
r32003 | return util.compengines['zlib'].decompressorreader(resp) | ||
Gregory Szorc
|
r30464 | |||
Peter Arrenbrecht
|
r17192 | return resp | ||
def _call(self, cmd, **args): | ||||
fp = self._callstream(cmd, **args) | ||||
try: | ||||
return fp.read() | ||||
finally: | ||||
# if using keepalive, allow connection to be reused | ||||
fp.close() | ||||
def _callpush(self, cmd, cg, **args): | ||||
# have to stream bundle to a temp file because we do not have | ||||
# http 1.1 chunked transfer. | ||||
types = self.capable('unbundle') | ||||
try: | ||||
types = types.split(',') | ||||
except AttributeError: | ||||
# servers older than d1b16a746db6 will send 'unbundle' as a | ||||
# boolean capability. They only support headerless/uncompressed | ||||
# bundles. | ||||
types = [""] | ||||
for x in types: | ||||
Martin von Zweigbergk
|
r28666 | if x in bundle2.bundletypes: | ||
Peter Arrenbrecht
|
r17192 | type = x | ||
break | ||||
Martin von Zweigbergk
|
r28666 | tempname = bundle2.writebundle(self.ui, cg, None, type) | ||
Peter Arrenbrecht
|
r17192 | fp = httpconnection.httpsendfile(self.ui, tempname, "rb") | ||
headers = {'Content-Type': 'application/mercurial-0.1'} | ||||
try: | ||||
Matt Mackall
|
r25085 | r = self._call(cmd, data=fp, headers=headers, **args) | ||
vals = r.split('\n', 1) | ||||
if len(vals) < 2: | ||||
raise error.ResponseError(_("unexpected response:"), r) | ||||
return vals | ||||
Gregory Szorc
|
r25660 | except socket.error as err: | ||
Matt Mackall
|
r25085 | if err.args[0] in (errno.ECONNRESET, errno.EPIPE): | ||
Pierre-Yves David
|
r26587 | raise error.Abort(_('push failed: %s') % err.args[1]) | ||
raise error.Abort(err.args[1]) | ||||
Peter Arrenbrecht
|
r17192 | finally: | ||
fp.close() | ||||
os.unlink(tempname) | ||||
Pierre-Yves David
|
r21074 | def _calltwowaystream(self, cmd, fp, **args): | ||
fh = None | ||||
Matt Harbison
|
r23086 | fp_ = None | ||
Pierre-Yves David
|
r21074 | filename = None | ||
try: | ||||
# dump bundle to disk | ||||
fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg") | ||||
Pulkit Goyal
|
r30924 | fh = os.fdopen(fd, pycompat.sysstr("wb")) | ||
Pierre-Yves David
|
r21074 | d = fp.read(4096) | ||
while d: | ||||
fh.write(d) | ||||
d = fp.read(4096) | ||||
fh.close() | ||||
# start http push | ||||
Matt Harbison
|
r23086 | fp_ = httpconnection.httpsendfile(self.ui, filename, "rb") | ||
Pierre-Yves David
|
r21074 | headers = {'Content-Type': 'application/mercurial-0.1'} | ||
Matt Harbison
|
r23086 | return self._callstream(cmd, data=fp_, headers=headers, **args) | ||
Pierre-Yves David
|
r21074 | finally: | ||
Matt Harbison
|
r23086 | if fp_ is not None: | ||
fp_.close() | ||||
Pierre-Yves David
|
r21074 | if fh is not None: | ||
fh.close() | ||||
os.unlink(filename) | ||||
Pierre-Yves David
|
r20905 | def _callcompressable(self, cmd, **args): | ||
Gregory Szorc
|
r30464 | return self._callstream(cmd, _compressible=True, **args) | ||
Peter Arrenbrecht
|
r17192 | |||
Mads Kiilerich
|
r21188 | def _abort(self, exception): | ||
raise exception | ||||
Peter Arrenbrecht
|
r17192 | class httpspeer(httppeer): | ||
def __init__(self, ui, path): | ||||
if not url.has_https: | ||||
Pierre-Yves David
|
r26587 | raise error.Abort(_('Python support for SSL and HTTPS ' | ||
Peter Arrenbrecht
|
r17192 | 'is not installed')) | ||
httppeer.__init__(self, ui, path) | ||||
def instance(ui, path, create): | ||||
if create: | ||||
Pierre-Yves David
|
r26587 | raise error.Abort(_('cannot create new http repository')) | ||
Peter Arrenbrecht
|
r17192 | try: | ||
if path.startswith('https:'): | ||||
inst = httpspeer(ui, path) | ||||
else: | ||||
inst = httppeer(ui, path) | ||||
try: | ||||
# Try to do useful work when checking compatibility. | ||||
# Usually saves a roundtrip since we want the caps anyway. | ||||
inst._fetchcaps() | ||||
except error.RepoError: | ||||
# No luck, try older compatibility check. | ||||
inst.between([(nullid, nullid)]) | ||||
return inst | ||||
Gregory Szorc
|
r25660 | except error.RepoError as httpexception: | ||
Peter Arrenbrecht
|
r17192 | try: | ||
r = statichttprepo.instance(ui, "static-" + path, create) | ||||
FUJIWARA Katsunori
|
r29241 | ui.note(_('(falling back to static-http)\n')) | ||
Peter Arrenbrecht
|
r17192 | return r | ||
except error.RepoError: | ||||
raise httpexception # use the original http RepoError instead | ||||