##// END OF EJS Templates
revset: remove "small" argument from "_optimize"...
revset: remove "small" argument from "_optimize" `_optimize` calculates weights of subtrees. "small" affects some weight calculation (either 1 or 0.5). The weights are now only useful in `and` optimization where we might swap two arguments and use `andsmally`. In the real world, it seems unlikely that revsets with weight of 0.5 or 1 matters the `and` order optimization. I think the important thing is to get weights of expensive revsets right (ex. `contains`). This patch removes the `small` argument to simplify the interface. As for choosing between 0.5 vs 1, things returning a single revision (`ancestor`, `string`) has a weight of 0.5. Things returning multiple revisions returns 1. This could be sometimes useful in the `andsmally` optimization, ex. (((:)-2) & expensive()) & ((1-2) & expensive()) ^^^ ^ ^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^ weight=1 weight=0.5 would have an `andsmally` optimization so `1-2` gets executed first, which seems to be desirable. Differential Revision: https://phab.mercurial-scm.org/D656

File last commit:

r33820:3c91cc0c default
r34273:53fb09c7 default
Show More
httppeer.py
464 lines | 16.2 KiB | text/x-python | PythonLexer
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 # httppeer.py - HTTP repository proxy classes for mercurial
#
# Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
Gregory Szorc
httppeer: use absolute_import
r25954 from __future__ import absolute_import
import errno
Augie Fackler
httppeer: add support for httppostargs when we're sending a file...
r33820 import io
Gregory Szorc
httppeer: use absolute_import
r25954 import os
import socket
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 import struct
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 import tempfile
Gregory Szorc
httppeer: use absolute_import
r25954
from .i18n import _
from .node import nullid
from . import (
Martin von Zweigbergk
bundle: move writebundle() from changegroup.py to bundle2.py (API)...
r28666 bundle2,
Gregory Szorc
httppeer: use absolute_import
r25954 error,
httpconnection,
Pulkit Goyal
py3: convert the mode argument of os.fdopen to unicodes (1 of 2)...
r30924 pycompat,
Gregory Szorc
httppeer: use absolute_import
r25954 statichttprepo,
url,
util,
wireproto,
)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Pulkit Goyal
py3: conditionalize httplib import...
r29455 httplib = util.httplib
timeless
pycompat: switch to util.urlreq/util.urlerr for py3 compat
r28883 urlerr = util.urlerr
urlreq = util.urlreq
Gregory Szorc
httppeer: extract code for HTTP header spanning...
r30759 def encodevalueinheaders(value, header, limit):
"""Encode a string value into multiple HTTP headers.
``value`` will be encoded into 1 or more HTTP headers with the names
``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
name + value will be at most ``limit`` bytes long.
Returns an iterable of 2-tuples consisting of header names and values.
"""
fmt = header + '-%s'
valuelen = limit - len(fmt % '000') - len(': \r\n')
result = []
n = 0
for i in xrange(0, len(value), valuelen):
n += 1
result.append((fmt % str(n), value[i:i + valuelen]))
return result
Gregory Szorc
httppeer: wrap HTTPResponse.read() globally...
r32002 def _wraphttpresponse(resp):
"""Wrap an HTTPResponse with common error handlers.
This ensures that any I/O from any consumer raises the appropriate
error and messaging.
"""
origread = resp.read
class readerproxy(resp.__class__):
def read(self, size=None):
try:
return origread(size)
except httplib.IncompleteRead as e:
# e.expected is an integer if length known or None otherwise.
if e.expected:
msg = _('HTTP request error (incomplete response; '
'expected %d bytes got %d)') % (e.expected,
len(e.partial))
else:
msg = _('HTTP request error (incomplete response)')
Gregory Szorc
error: rename RichIOError to PeerTransportError...
r32023 raise error.PeerTransportError(
Gregory Szorc
httppeer: wrap HTTPResponse.read() globally...
r32002 msg,
hint=_('this may be an intermittent network failure; '
'if the error persists, consider contacting the '
'network or server operator'))
except httplib.HTTPException as e:
Gregory Szorc
error: rename RichIOError to PeerTransportError...
r32023 raise error.PeerTransportError(
Gregory Szorc
httppeer: wrap HTTPResponse.read() globally...
r32002 _('HTTP request error (%s)') % e,
FUJIWARA Katsunori
httppeer: unify hint message for PeerTransportError...
r32087 hint=_('this may be an intermittent network failure; '
Gregory Szorc
httppeer: wrap HTTPResponse.read() globally...
r32002 'if the error persists, consider contacting the '
'network or server operator'))
resp.__class__ = readerproxy
Augie Fackler
httppeer: add support for httppostargs when we're sending a file...
r33820 class _multifile(object):
def __init__(self, *fileobjs):
for f in fileobjs:
if not util.safehasattr(f, 'length'):
raise ValueError(
'_multifile only supports file objects that '
'have a length but this one does not:', type(f), f)
self._fileobjs = fileobjs
self._index = 0
@property
def length(self):
return sum(f.length for f in self._fileobjs)
def read(self, amt=None):
if amt <= 0:
return ''.join(f.read() for f in self._fileobjs)
parts = []
while amt and self._index < len(self._fileobjs):
parts.append(self._fileobjs[self._index].read(amt))
got = len(parts[-1])
if got < amt:
self._index += 1
amt -= got
return ''.join(parts)
def seek(self, offset, whence=os.SEEK_SET):
if whence != os.SEEK_SET:
raise NotImplementedError(
'_multifile does not support anything other'
' than os.SEEK_SET for whence on seek()')
if offset != 0:
raise NotImplementedError(
'_multifile only supports seeking to start, but that '
'could be fixed if you need it')
for f in self._fileobjs:
f.seek(0)
self._index = 0
Gregory Szorc
wireproto: use new peer interface...
r33805 class httppeer(wireproto.wirepeer):
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 def __init__(self, ui, path):
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 self._path = path
self._caps = None
self._urlopener = None
self._requestbuilder = None
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 u = util.url(path)
if u.query or u.fragment:
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('unsupported URL component: "%s"') %
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 (u.query or u.fragment))
# urllib cannot handle URLs with embedded user or passwd
self._url, authinfo = u.authinfo()
Gregory Szorc
httppeer: use peer interface...
r33804 self._ui = ui
ui.debug('using %s\n' % self._url)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 self._urlopener = url.opener(ui, authinfo)
self._requestbuilder = urlreq.request
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
def __del__(self):
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 urlopener = getattr(self, '_urlopener', None)
Mads Kiilerich
httppeer: make __del__ access to self.urlopener more safe...
r30241 if urlopener:
for h in urlopener.handlers:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 h.close()
getattr(h, "close_all", lambda : None)()
Gregory Szorc
httppeer: use peer interface...
r33804 # Begin of _basepeer interface.
@util.propertycache
def ui(self):
return self._ui
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 def url(self):
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 return self._path
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Gregory Szorc
httppeer: use peer interface...
r33804 def local(self):
return None
def peer(self):
return self
def canpush(self):
return True
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Gregory Szorc
httppeer: use peer interface...
r33804 def close(self):
pass
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Gregory Szorc
httppeer: use peer interface...
r33804 # End of _basepeer interface.
# Begin of _basewirepeer interface.
def capabilities(self):
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 if self._caps is None:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 try:
self._fetchcaps()
except error.RepoError:
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 self._caps = set()
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 self.ui.debug('capabilities: %s\n' %
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 (' '.join(self._caps or ['none'])))
return self._caps
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Gregory Szorc
httppeer: use peer interface...
r33804 # End of _basewirepeer interface.
# look up capabilities only when needed
def _fetchcaps(self):
self._caps = set(self._call('capabilities').split())
Gregory Szorc
httppeer: do decompression inside _callstream...
r30464 def _callstream(self, cmd, _compressible=False, **args):
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 if cmd == 'pushkey':
args['data'] = ''
data = args.pop('data', None)
headers = args.pop('headers', {})
self.ui.debug("sending %s command\n" % cmd)
q = [('cmd', cmd)]
headersize = 0
Gregory Szorc
httppeer: assign Vary request header last...
r30564 varyheaders = []
Augie Fackler
http: support sending hgargs via POST body instead of in GET or headers...
r28530 # Important: don't use self.capable() here or else you end up
# with infinite recursion when trying to look up capabilities
# for the first time.
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 postargsok = self._caps is not None and 'httppostargs' in self._caps
Augie Fackler
httppeer: add support for httppostargs when we're sending a file...
r33820 if postargsok and args:
timeless
pycompat: switch to util.urlreq/util.urlerr for py3 compat
r28883 strargs = urlreq.urlencode(sorted(args.items()))
Augie Fackler
httppeer: add support for httppostargs when we're sending a file...
r33820 if not data:
data = strargs
else:
if isinstance(data, basestring):
i = io.BytesIO(data)
i.length = len(data)
data = i
argsio = io.BytesIO(strargs)
argsio.length = len(strargs)
data = _multifile(argsio, data)
headers['X-HgArgs-Post'] = len(strargs)
Augie Fackler
http: support sending hgargs via POST body instead of in GET or headers...
r28530 else:
Augie Fackler
httppeer: indent existing argument handling with if True...
r28485 if len(args) > 0:
httpheader = self.capable('httpheader')
if httpheader:
headersize = int(httpheader.split(',', 1)[0])
if headersize > 0:
# The headers can typically carry more data than the URL.
timeless
pycompat: switch to util.urlreq/util.urlerr for py3 compat
r28883 encargs = urlreq.urlencode(sorted(args.items()))
Gregory Szorc
httppeer: extract code for HTTP header spanning...
r30759 for header, value in encodevalueinheaders(encargs, 'X-HgArg',
headersize):
headers[header] = value
Augie Fackler
httppeer: compute header names only once...
r28486 varyheaders.append(header)
Augie Fackler
httppeer: indent existing argument handling with if True...
r28485 else:
q += sorted(args.items())
timeless
pycompat: switch to util.urlreq/util.urlerr for py3 compat
r28883 qs = '?%s' % urlreq.urlencode(q)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 cu = "%s%s" % (self._url, qs)
Augie Fackler
httppeer: move size computation later in _callstream...
r28484 size = 0
if util.safehasattr(data, 'length'):
size = data.length
elif data is not None:
size = len(data)
Jun Wu
codemod: register core configitems using a script...
r33499 if size and self.ui.configbool('ui', 'usehttp2'):
Augie Fackler
httppeer: move size computation later in _callstream...
r28484 headers['Expect'] = '100-Continue'
headers['X-HgHttp2'] = '1'
if data is not None and 'Content-Type' not in headers:
headers['Content-Type'] = 'application/mercurial-0.1'
Gregory Szorc
httppeer: assign Vary request header last...
r30564
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 # Tell the server we accept application/mercurial-0.2 and multiple
# compression formats if the server is capable of emitting those
# payloads.
protoparams = []
mediatypes = set()
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 if self._caps is not None:
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 mt = self.capable('httpmediatype')
if mt:
protoparams.append('0.1')
mediatypes = set(mt.split(','))
if '0.2tx' in mediatypes:
protoparams.append('0.2')
if '0.2tx' in mediatypes and self.capable('compression'):
# We /could/ compare supported compression formats and prune
# non-mutually supported or error if nothing is mutually supported.
# For now, send the full list to the server and have it error.
comps = [e.wireprotosupport().name for e in
util.compengines.supportedwireengines(util.CLIENTROLE)]
protoparams.append('comp=%s' % ','.join(comps))
if protoparams:
protoheaders = encodevalueinheaders(' '.join(protoparams),
'X-HgProto',
headersize or 1024)
for header, value in protoheaders:
headers[header] = value
varyheaders.append(header)
Gregory Szorc
httppeer: don't send empty Vary request header...
r32022 if varyheaders:
headers['Vary'] = ','.join(varyheaders)
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 req = self._requestbuilder(cu, data, headers)
Gregory Szorc
httppeer: assign Vary request header last...
r30564
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 if data is not None:
self.ui.debug("sending %s bytes\n" % size)
req.add_unredirected_header('Content-Length', '%d' % size)
try:
Gregory Szorc
httppeer: make several instance attributes internal (API)...
r33671 resp = self._urlopener.open(req)
timeless
pycompat: switch to util.urlreq/util.urlerr for py3 compat
r28883 except urlerr.httperror as inst:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 if inst.code == 401:
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('authorization failed'))
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 raise
Gregory Szorc
global: mass rewrite to use modern exception syntax...
r25660 except httplib.HTTPException as inst:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 self.ui.debug('http error while sending %s command\n' % cmd)
self.ui.traceback()
raise IOError(None, inst)
Gregory Szorc
httppeer: wrap HTTPResponse.read() globally...
r32002
# Insert error handlers for common I/O failures.
_wraphttpresponse(resp)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 # record the url we got redirected to
resp_url = resp.geturl()
if resp_url.endswith(qs):
resp_url = resp_url[:-len(qs)]
if self._url.rstrip('/') != resp_url.rstrip('/'):
if not self.ui.quiet:
self.ui.warn(_('real URL is %s\n') % resp_url)
self._url = resp_url
try:
proto = resp.getheader('content-type')
except AttributeError:
proto = resp.headers.get('content-type', '')
safeurl = util.hidepassword(self._url)
if proto.startswith('application/hg-error'):
raise error.OutOfBandError(resp.read())
# accept old "text/plain" and "application/hg-changegroup" for now
if not (proto.startswith('application/mercurial-') or
Matt Mackall
httppeer: improve protocol check...
r18737 (proto.startswith('text/plain')
and not resp.headers.get('content-length')) or
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 proto.startswith('application/hg-changegroup')):
self.ui.debug("requested URL: '%s'\n" % util.hidepassword(cu))
raise error.RepoError(
_("'%s' does not appear to be an hg repository:\n"
"---%%<--- (%s)\n%s\n---%%<---\n")
Matt Mackall
httppeer: avoid large dumps when we don't see an hgweb repo...
r18738 % (safeurl, proto or 'no content-type', resp.read(1024)))
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
if proto.startswith('application/mercurial-'):
try:
version = proto.split('-', 1)[1]
version_info = tuple([int(n) for n in version.split('.')])
except ValueError:
raise error.RepoError(_("'%s' sent a broken Content-Type "
"header (%s)") % (safeurl, proto))
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763
Gregory Szorc
httppeer: eliminate decompressresponse() proxy...
r32003 # TODO consider switching to a decompression reader that uses
# generators.
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 if version_info == (0, 1):
if _compressible:
Gregory Szorc
httppeer: eliminate decompressresponse() proxy...
r32003 return util.compengines['zlib'].decompressorreader(resp)
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 return resp
elif version_info == (0, 2):
# application/mercurial-0.2 always identifies the compression
# engine in the payload header.
elen = struct.unpack('B', resp.read(1))[0]
ename = resp.read(elen)
engine = util.compengines.forwiretype(ename)
Gregory Szorc
httppeer: eliminate decompressresponse() proxy...
r32003 return engine.decompressorreader(resp)
Gregory Szorc
httppeer: advertise and support application/mercurial-0.2...
r30763 else:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 raise error.RepoError(_("'%s' uses newer protocol %s") %
(safeurl, version))
Gregory Szorc
httppeer: do decompression inside _callstream...
r30464 if _compressible:
Gregory Szorc
httppeer: eliminate decompressresponse() proxy...
r32003 return util.compengines['zlib'].decompressorreader(resp)
Gregory Szorc
httppeer: do decompression inside _callstream...
r30464
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 return resp
def _call(self, cmd, **args):
fp = self._callstream(cmd, **args)
try:
return fp.read()
finally:
# if using keepalive, allow connection to be reused
fp.close()
def _callpush(self, cmd, cg, **args):
# have to stream bundle to a temp file because we do not have
# http 1.1 chunked transfer.
types = self.capable('unbundle')
try:
types = types.split(',')
except AttributeError:
# servers older than d1b16a746db6 will send 'unbundle' as a
# boolean capability. They only support headerless/uncompressed
# bundles.
types = [""]
for x in types:
Martin von Zweigbergk
bundle: move writebundle() from changegroup.py to bundle2.py (API)...
r28666 if x in bundle2.bundletypes:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 type = x
break
Martin von Zweigbergk
bundle: move writebundle() from changegroup.py to bundle2.py (API)...
r28666 tempname = bundle2.writebundle(self.ui, cg, None, type)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
headers = {'Content-Type': 'application/mercurial-0.1'}
try:
Matt Mackall
httppeer: use try/except/finally
r25085 r = self._call(cmd, data=fp, headers=headers, **args)
vals = r.split('\n', 1)
if len(vals) < 2:
raise error.ResponseError(_("unexpected response:"), r)
return vals
Gregory Szorc
global: mass rewrite to use modern exception syntax...
r25660 except socket.error as err:
Matt Mackall
httppeer: use try/except/finally
r25085 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('push failed: %s') % err.args[1])
raise error.Abort(err.args[1])
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 finally:
fp.close()
os.unlink(tempname)
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 def _calltwowaystream(self, cmd, fp, **args):
fh = None
Matt Harbison
httppeer: close the temporary bundle file after two-way streaming it...
r23086 fp_ = None
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 filename = None
try:
# dump bundle to disk
fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg")
Pulkit Goyal
py3: convert the mode argument of os.fdopen to unicodes (1 of 2)...
r30924 fh = os.fdopen(fd, pycompat.sysstr("wb"))
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 d = fp.read(4096)
while d:
fh.write(d)
d = fp.read(4096)
fh.close()
# start http push
Matt Harbison
httppeer: close the temporary bundle file after two-way streaming it...
r23086 fp_ = httpconnection.httpsendfile(self.ui, filename, "rb")
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 headers = {'Content-Type': 'application/mercurial-0.1'}
Matt Harbison
httppeer: close the temporary bundle file after two-way streaming it...
r23086 return self._callstream(cmd, data=fp_, headers=headers, **args)
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 finally:
Matt Harbison
httppeer: close the temporary bundle file after two-way streaming it...
r23086 if fp_ is not None:
fp_.close()
Pierre-Yves David
httppeer: support for _calltwowaystream...
r21074 if fh is not None:
fh.close()
os.unlink(filename)
Pierre-Yves David
wireproto: drop the _decompress method in favor a new call type...
r20905 def _callcompressable(self, cmd, **args):
Gregory Szorc
httppeer: do decompression inside _callstream...
r30464 return self._callstream(cmd, _compressible=True, **args)
Peter Arrenbrecht
peer: introduce real peer classes...
r17192
Mads Kiilerich
httppeer: reintroduce _abort that accidentally was removed in 167047ba3cfa...
r21188 def _abort(self, exception):
raise exception
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 class httpspeer(httppeer):
def __init__(self, ui, path):
if not url.has_https:
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('Python support for SSL and HTTPS '
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 'is not installed'))
httppeer.__init__(self, ui, path)
def instance(ui, path, create):
if create:
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('cannot create new http repository'))
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 try:
if path.startswith('https:'):
inst = httpspeer(ui, path)
else:
inst = httppeer(ui, path)
try:
# Try to do useful work when checking compatibility.
# Usually saves a roundtrip since we want the caps anyway.
inst._fetchcaps()
except error.RepoError:
# No luck, try older compatibility check.
inst.between([(nullid, nullid)])
return inst
Gregory Szorc
global: mass rewrite to use modern exception syntax...
r25660 except error.RepoError as httpexception:
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 try:
r = statichttprepo.instance(ui, "static-" + path, create)
FUJIWARA Katsunori
httppeer: make a message translatable...
r29241 ui.note(_('(falling back to static-http)\n'))
Peter Arrenbrecht
peer: introduce real peer classes...
r17192 return r
except error.RepoError:
raise httpexception # use the original http RepoError instead