httprepo.py
458 lines
| 16.3 KiB
| text/x-python
|
PythonLexer
/ mercurial / httprepo.py
|
r1089 | # httprepo.py - HTTP repository proxy classes for mercurial | ||
# | ||||
|
r2859 | # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com> | ||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
|
r1089 | # | ||
# This software may be used and distributed according to the terms | ||||
# of the GNU General Public License, incorporated herein by reference. | ||||
|
r6211 | from node import bin, hex | ||
from remoterepo import remoterepository | ||||
|
r3891 | from i18n import _ | ||
|
r5176 | import repo, os, urllib, urllib2, urlparse, zlib, util, httplib | ||
|
r6285 | import errno, keepalive, socket, changegroup, version | ||
|
r1089 | |||
|
r2447 | class passwordmgr(urllib2.HTTPPasswordMgrWithDefaultRealm): | ||
|
r2281 | def __init__(self, ui): | ||
|
r2447 | urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self) | ||
|
r2281 | self.ui = ui | ||
def find_user_password(self, realm, authuri): | ||||
|
r2447 | authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password( | ||
|
r2281 | self, realm, authuri) | ||
|
r2556 | user, passwd = authinfo | ||
if user and passwd: | ||||
return (user, passwd) | ||||
|
r2281 | |||
|
r2446 | if not self.ui.interactive: | ||
|
r2337 | raise util.Abort(_('http authorization required')) | ||
|
r2281 | self.ui.write(_("http authorization required\n")) | ||
self.ui.status(_("realm: %s\n") % realm) | ||||
|
r2556 | if user: | ||
self.ui.status(_("user: %s\n") % user) | ||||
else: | ||||
user = self.ui.prompt(_("user:"), default=None) | ||||
if not passwd: | ||||
passwd = self.ui.getpass() | ||||
|
r2281 | |||
self.add_password(realm, authuri, user, passwd) | ||||
return (user, passwd) | ||||
|
r2337 | def netlocsplit(netloc): | ||
'''split [user[:passwd]@]host[:port] into 4-tuple.''' | ||||
a = netloc.find('@') | ||||
if a == -1: | ||||
user, passwd = None, None | ||||
else: | ||||
userpass, netloc = netloc[:a], netloc[a+1:] | ||||
c = userpass.find(':') | ||||
if c == -1: | ||||
user, passwd = urllib.unquote(userpass), None | ||||
else: | ||||
user = urllib.unquote(userpass[:c]) | ||||
passwd = urllib.unquote(userpass[c+1:]) | ||||
c = netloc.find(':') | ||||
if c == -1: | ||||
host, port = netloc, None | ||||
else: | ||||
host, port = netloc[:c], netloc[c+1:] | ||||
return host, port, user, passwd | ||||
def netlocunsplit(host, port, user=None, passwd=None): | ||||
'''turn host, port, user, passwd into [user[:passwd]@]host[:port].''' | ||||
if port: | ||||
hostport = host + ':' + port | ||||
else: | ||||
hostport = host | ||||
if user: | ||||
if passwd: | ||||
userpass = urllib.quote(user) + ':' + urllib.quote(passwd) | ||||
else: | ||||
userpass = urllib.quote(user) | ||||
return userpass + '@' + hostport | ||||
return hostport | ||||
|
r4226 | # work around a bug in Python < 2.4.2 | ||
# (it leaves a "\n" at the end of Proxy-authorization headers) | ||||
class request(urllib2.Request): | ||||
def add_header(self, key, val): | ||||
if key.lower() == 'proxy-authorization': | ||||
val = val.strip() | ||||
return urllib2.Request.add_header(self, key, val) | ||||
|
r4025 | class httpsendfile(file): | ||
def __len__(self): | ||||
return os.fstat(self.fileno()).st_size | ||||
|
r2465 | |||
|
r4025 | def _gen_sendfile(connection): | ||
def _sendfile(self, data): | ||||
# send a file | ||||
if isinstance(data, httpsendfile): | ||||
|
r2465 | # if auth required, some data sent twice, so rewind here | ||
data.seek(0) | ||||
for chunk in util.filechunkiter(data): | ||||
|
r6289 | connection.sendall(self, chunk) | ||
|
r4025 | else: | ||
|
r6289 | connection.sendall(self, data) | ||
|
r4025 | return _sendfile | ||
class httpconnection(keepalive.HTTPConnection): | ||||
# must be able to send big bundle as stream. | ||||
send = _gen_sendfile(keepalive.HTTPConnection) | ||||
|
r2465 | |||
|
r5983 | class httphandler(keepalive.HTTPHandler): | ||
|
r2465 | def http_open(self, req): | ||
return self.do_open(httpconnection, req) | ||||
|
r5982 | def __del__(self): | ||
self.close_all() | ||||
|
r2569 | has_https = hasattr(urllib2, 'HTTPSHandler') | ||
if has_https: | ||||
class httpsconnection(httplib.HTTPSConnection): | ||||
response_class = keepalive.HTTPResponse | ||||
# must be able to send big bundle as stream. | ||||
|
r4025 | send = _gen_sendfile(httplib.HTTPSConnection) | ||
|
r2557 | |||
|
r5983 | class httpshandler(keepalive.KeepAliveHandler, urllib2.HTTPSHandler): | ||
|
r2569 | def https_open(self, req): | ||
return self.do_open(httpsconnection, req) | ||||
|
r2557 | |||
|
r4678 | # In python < 2.5 AbstractDigestAuthHandler raises a ValueError if | ||
# it doesn't know about the auth type requested. This can happen if | ||||
# somebody is using BasicAuth and types a bad password. | ||||
class httpdigestauthhandler(urllib2.HTTPDigestAuthHandler): | ||||
def http_error_auth_reqed(self, auth_header, host, req, headers): | ||||
try: | ||||
return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed( | ||||
self, auth_header, host, req, headers) | ||||
except ValueError, inst: | ||||
arg = inst.args[0] | ||||
if arg.startswith("AbstractDigestAuthHandler doesn't know "): | ||||
return | ||||
raise | ||||
|
r3661 | def zgenerator(f): | ||
zd = zlib.decompressobj() | ||||
try: | ||||
for chunk in util.filechunkiter(f): | ||||
yield zd.decompress(chunk) | ||||
except httplib.HTTPException, inst: | ||||
raise IOError(None, _('connection ended unexpectedly')) | ||||
yield zd.flush() | ||||
|
r5066 | _safe = ('abcdefghijklmnopqrstuvwxyz' | ||
'ABCDEFGHIJKLMNOPQRSTUVWXYZ' | ||||
'0123456789' '_.-/') | ||||
_safeset = None | ||||
_hex = None | ||||
def quotepath(path): | ||||
'''quote the path part of a URL | ||||
This is similar to urllib.quote, but it also tries to avoid | ||||
quoting things twice (inspired by wget): | ||||
>>> quotepath('abc def') | ||||
'abc%20def' | ||||
>>> quotepath('abc%20def') | ||||
'abc%20def' | ||||
>>> quotepath('abc%20 def') | ||||
'abc%20%20def' | ||||
>>> quotepath('abc def%20') | ||||
'abc%20def%20' | ||||
>>> quotepath('abc def%2') | ||||
'abc%20def%252' | ||||
>>> quotepath('abc def%') | ||||
'abc%20def%25' | ||||
''' | ||||
global _safeset, _hex | ||||
if _safeset is None: | ||||
_safeset = util.set(_safe) | ||||
_hex = util.set('abcdefABCDEF0123456789') | ||||
l = list(path) | ||||
for i in xrange(len(l)): | ||||
c = l[i] | ||||
if c == '%' and i + 2 < len(l) and (l[i+1] in _hex and l[i+2] in _hex): | ||||
pass | ||||
elif c not in _safeset: | ||||
l[i] = '%%%02X' % ord(c) | ||||
return ''.join(l) | ||||
|
r1089 | class httprepository(remoterepository): | ||
def __init__(self, ui, path): | ||||
|
r2673 | self.path = path | ||
|
r2442 | self.caps = None | ||
|
r4132 | self.handler = None | ||
|
r2337 | scheme, netloc, urlpath, query, frag = urlparse.urlsplit(path) | ||
if query or frag: | ||||
raise util.Abort(_('unsupported URL component: "%s"') % | ||||
(query or frag)) | ||||
|
r5066 | if not urlpath: | ||
urlpath = '/' | ||||
urlpath = quotepath(urlpath) | ||||
|
r2337 | host, port, user, passwd = netlocsplit(netloc) | ||
# urllib cannot handle URLs with embedded user or passwd | ||||
|
r2673 | self._url = urlparse.urlunsplit((scheme, netlocunsplit(host, port), | ||
urlpath, '', '')) | ||||
|
r1089 | self.ui = ui | ||
|
r5066 | self.ui.debug(_('using %s\n') % self._url) | ||
|
r2337 | |||
proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy') | ||||
|
r3131 | # XXX proxyauthinfo = None | ||
|
r5982 | handlers = [httphandler()] | ||
|
r5983 | if has_https: | ||
handlers.append(httpshandler()) | ||||
|
r2337 | |||
if proxyurl: | ||||
# proxy can be proper url or host[:port] | ||||
if not (proxyurl.startswith('http:') or | ||||
proxyurl.startswith('https:')): | ||||
proxyurl = 'http://' + proxyurl + '/' | ||||
snpqf = urlparse.urlsplit(proxyurl) | ||||
proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf | ||||
hpup = netlocsplit(proxynetloc) | ||||
proxyhost, proxyport, proxyuser, proxypasswd = hpup | ||||
if not proxyuser: | ||||
proxyuser = ui.config("http_proxy", "user") | ||||
proxypasswd = ui.config("http_proxy", "passwd") | ||||
|
r1089 | |||
|
r2337 | # see if we should use a proxy for this url | ||
no_list = [ "localhost", "127.0.0.1" ] | ||||
|
r2501 | no_list.extend([p.lower() for | ||
p in ui.configlist("http_proxy", "no")]) | ||||
|
r2337 | no_list.extend([p.strip().lower() for | ||
p in os.getenv("no_proxy", '').split(',') | ||||
if p.strip()]) | ||||
# "http_proxy.always" config is for running tests on localhost | ||||
if (not ui.configbool("http_proxy", "always") and | ||||
host.lower() in no_list): | ||||
|
r5476 | # avoid auto-detection of proxy settings by appending | ||
# a ProxyHandler with no proxies defined. | ||||
handlers.append(urllib2.ProxyHandler({})) | ||||
|
r2337 | ui.debug(_('disabling proxy for %s\n') % host) | ||
else: | ||||
proxyurl = urlparse.urlunsplit(( | ||||
proxyscheme, netlocunsplit(proxyhost, proxyport, | ||||
proxyuser, proxypasswd or ''), | ||||
proxypath, proxyquery, proxyfrag)) | ||||
|
r3608 | handlers.append(urllib2.ProxyHandler({scheme: proxyurl})) | ||
|
r3223 | ui.debug(_('proxying through http://%s:%s\n') % | ||
|
r3170 | (proxyhost, proxyport)) | ||
|
r1089 | |||
|
r2337 | # urllib2 takes proxy values from the environment and those | ||
# will take precedence if found, so drop them | ||||
|
r1089 | for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]: | ||
try: | ||||
|
r5915 | if env in os.environ: | ||
|
r1089 | del os.environ[env] | ||
except OSError: | ||||
pass | ||||
|
r2337 | passmgr = passwordmgr(ui) | ||
if user: | ||||
|
r2556 | ui.debug(_('http auth: user %s, password %s\n') % | ||
(user, passwd and '*' * len(passwd) or 'not set')) | ||||
|
r5526 | netloc = host | ||
if port: | ||||
netloc += ':' + port | ||||
# Python < 2.4.3 uses only the netloc to search for a password | ||||
passmgr.add_password(None, (self._url, netloc), user, passwd or '') | ||||
|
r1089 | |||
|
r3608 | handlers.extend((urllib2.HTTPBasicAuthHandler(passmgr), | ||
|
r4678 | httpdigestauthhandler(passmgr))) | ||
|
r3608 | opener = urllib2.build_opener(*handlers) | ||
|
r2281 | |||
|
r1359 | # 1.0 here is the _protocol_ version | ||
|
r6285 | opener.addheaders = [('User-agent', version.get_useragent())] | ||
|
r1089 | urllib2.install_opener(opener) | ||
|
r4516 | |||
|
r2673 | def url(self): | ||
return self.path | ||||
|
r2442 | # look up capabilities only when needed | ||
def get_caps(self): | ||||
if self.caps is None: | ||||
try: | ||||
|
r5258 | self.caps = util.set(self.do_read('capabilities').split()) | ||
|
r5176 | except repo.RepoError: | ||
|
r5258 | self.caps = util.set() | ||
|
r2465 | self.ui.debug(_('capabilities: %s\n') % | ||
(' '.join(self.caps or ['none']))) | ||||
|
r2442 | return self.caps | ||
capabilities = property(get_caps) | ||||
|
r1870 | def lock(self): | ||
raise util.Abort(_('operation not supported over http')) | ||||
|
r1089 | def do_cmd(self, cmd, **args): | ||
|
r2465 | data = args.pop('data', None) | ||
headers = args.pop('headers', {}) | ||||
|
r1402 | self.ui.debug(_("sending %s command\n") % cmd) | ||
|
r1089 | q = {"cmd": cmd} | ||
q.update(args) | ||||
|
r3562 | qs = '?%s' % urllib.urlencode(q) | ||
cu = "%s%s" % (self._url, qs) | ||||
|
r2294 | try: | ||
|
r3567 | if data: | ||
|
r5333 | self.ui.debug(_("sending %s bytes\n") % len(data)) | ||
|
r4226 | resp = urllib2.urlopen(request(cu, data, headers)) | ||
|
r2467 | except urllib2.HTTPError, inst: | ||
if inst.code == 401: | ||||
raise util.Abort(_('authorization failed')) | ||||
raise | ||||
|
r2294 | except httplib.HTTPException, inst: | ||
|
r2336 | self.ui.debug(_('http error while sending %s command\n') % cmd) | ||
self.ui.print_exc() | ||||
raise IOError(None, inst) | ||||
|
r3399 | except IndexError: | ||
# this only happens with Python 2.3, later versions raise URLError | ||||
raise util.Abort(_('http error, possibly caused by proxy setting')) | ||||
|
r3562 | # record the url we got redirected to | ||
|
r3570 | resp_url = resp.geturl() | ||
if resp_url.endswith(qs): | ||||
resp_url = resp_url[:-len(qs)] | ||||
if self._url != resp_url: | ||||
self.ui.status(_('real URL is %s\n') % resp_url) | ||||
self._url = resp_url | ||||
|
r2435 | try: | ||
proto = resp.getheader('content-type') | ||||
except AttributeError: | ||||
proto = resp.headers['content-type'] | ||||
|
r1089 | |||
# accept old "text/plain" and "application/hg-changegroup" for now | ||||
|
r4633 | if not (proto.startswith('application/mercurial-') or | ||
proto.startswith('text/plain') or | ||||
proto.startswith('application/hg-changegroup')): | ||||
|
r4739 | self.ui.debug(_("Requested URL: '%s'\n") % cu) | ||
|
r5176 | raise repo.RepoError(_("'%s' does not appear to be an hg repository") | ||
|
r4633 | % self._url) | ||
|
r1089 | |||
|
r4012 | if proto.startswith('application/mercurial-'): | ||
try: | ||||
|
r4356 | version = proto.split('-', 1)[1] | ||
version_info = tuple([int(n) for n in version.split('.')]) | ||||
|
r4012 | except ValueError: | ||
|
r5930 | raise repo.RepoError(_("'%s' sent a broken Content-Type " | ||
|
r4012 | "header (%s)") % (self._url, proto)) | ||
|
r4356 | if version_info > (0, 1): | ||
|
r5176 | raise repo.RepoError(_("'%s' uses newer protocol %s") % | ||
|
r2673 | (self._url, version)) | ||
|
r1089 | |||
return resp | ||||
|
r2435 | def do_read(self, cmd, **args): | ||
fp = self.do_cmd(cmd, **args) | ||||
try: | ||||
return fp.read() | ||||
finally: | ||||
# if using keepalive, allow connection to be reused | ||||
fp.close() | ||||
|
r3444 | def lookup(self, key): | ||
|
r5259 | self.requirecap('lookup', _('look up remote revision')) | ||
|
r3445 | d = self.do_cmd("lookup", key = key).read() | ||
success, data = d[:-1].split(' ', 1) | ||||
if int(success): | ||||
return bin(data) | ||||
|
r5176 | raise repo.RepoError(data) | ||
|
r3444 | |||
|
r1089 | def heads(self): | ||
|
r2435 | d = self.do_read("heads") | ||
|
r1089 | try: | ||
return map(bin, d[:-1].split(" ")) | ||||
except: | ||||
|
r3565 | raise util.UnexpectedOutput(_("unexpected response:"), d) | ||
|
r1089 | |||
def branches(self, nodes): | ||||
n = " ".join(map(hex, nodes)) | ||||
|
r2435 | d = self.do_read("branches", nodes=n) | ||
|
r1089 | try: | ||
br = [ tuple(map(bin, b.split(" "))) for b in d.splitlines() ] | ||||
return br | ||||
except: | ||||
|
r3565 | raise util.UnexpectedOutput(_("unexpected response:"), d) | ||
|
r1089 | |||
def between(self, pairs): | ||||
n = "\n".join(["-".join(map(hex, p)) for p in pairs]) | ||||
|
r2435 | d = self.do_read("between", pairs=n) | ||
|
r1089 | try: | ||
p = [ l and map(bin, l.split(" ")) or [] for l in d.splitlines() ] | ||||
return p | ||||
except: | ||||
|
r3565 | raise util.UnexpectedOutput(_("unexpected response:"), d) | ||
|
r1089 | |||
|
r1736 | def changegroup(self, nodes, kind): | ||
|
r1089 | n = " ".join(map(hex, nodes)) | ||
f = self.do_cmd("changegroup", roots=n) | ||||
|
r3661 | return util.chunkbuffer(zgenerator(f)) | ||
|
r3444 | |||
def changegroupsubset(self, bases, heads, source): | ||||
|
r5259 | self.requirecap('changegroupsubset', _('look up remote changes')) | ||
|
r3444 | baselst = " ".join([hex(n) for n in bases]) | ||
headlst = " ".join([hex(n) for n in heads]) | ||||
f = self.do_cmd("changegroupsubset", bases=baselst, heads=headlst) | ||||
|
r3661 | return util.chunkbuffer(zgenerator(f)) | ||
|
r1089 | |||
|
r2439 | def unbundle(self, cg, heads, source): | ||
|
r2465 | # have to stream bundle to a temp file because we do not have | ||
# http 1.1 chunked transfer. | ||||
|
r3662 | type = "" | ||
types = self.capable('unbundle') | ||||
|
r3703 | # servers older than d1b16a746db6 will send 'unbundle' as a | ||
# boolean capability | ||||
try: | ||||
types = types.split(',') | ||||
except AttributeError: | ||||
types = [""] | ||||
|
r3662 | if types: | ||
|
r3703 | for x in types: | ||
|
r3662 | if x in changegroup.bundletypes: | ||
type = x | ||||
break | ||||
|
r3613 | |||
|
r3662 | tempname = changegroup.writebundle(cg, None, type) | ||
|
r4025 | fp = httpsendfile(tempname, "rb") | ||
|
r2465 | try: | ||
try: | ||||
|
r2467 | rfp = self.do_cmd( | ||
'unbundle', data=fp, | ||||
|
r5930 | headers={'Content-Type': 'application/octet-stream'}, | ||
|
r2467 | heads=' '.join(map(hex, heads))) | ||
try: | ||||
ret = int(rfp.readline()) | ||||
self.ui.write(rfp.read()) | ||||
return ret | ||||
finally: | ||||
rfp.close() | ||||
except socket.error, err: | ||||
if err[0] in (errno.ECONNRESET, errno.EPIPE): | ||||
|
r3072 | raise util.Abort(_('push failed: %s') % err[1]) | ||
|
r2467 | raise util.Abort(err[1]) | ||
|
r2465 | finally: | ||
fp.close() | ||||
os.unlink(tempname) | ||||
|
r2439 | |||
|
r2612 | def stream_out(self): | ||
return self.do_cmd('stream_out') | ||||
|
r1089 | class httpsrepository(httprepository): | ||
|
r2569 | def __init__(self, ui, path): | ||
if not has_https: | ||||
raise util.Abort(_('Python support for SSL and HTTPS ' | ||||
'is not installed')) | ||||
httprepository.__init__(self, ui, path) | ||||
|
r2740 | |||
def instance(ui, path, create): | ||||
if create: | ||||
raise util.Abort(_('cannot create new http repository')) | ||||
if path.startswith('https:'): | ||||
return httpsrepository(ui, path) | ||||
return httprepository(ui, path) | ||||