url.py
728 lines
| 24.2 KiB
| text/x-python
|
PythonLexer
/ mercurial / url.py
|
r7270 | # url.py - HTTP handling for mercurial | ||
# | ||||
# Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com> | ||||
# Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br> | ||||
# Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> | ||||
# | ||||
|
r8225 | # This software may be used and distributed according to the terms of the | ||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r7270 | |||
|
r25990 | from __future__ import absolute_import | ||
import base64 | ||||
import os | ||||
import socket | ||||
|
r37118 | import sys | ||
|
r25990 | |||
from .i18n import _ | ||||
|
r43359 | from .pycompat import getattr | ||
|
r25990 | from . import ( | ||
|
r30820 | encoding, | ||
|
r26587 | error, | ||
|
r25990 | httpconnection as httpconnectionmod, | ||
keepalive, | ||||
|
r34429 | pycompat, | ||
|
r25990 | sslutil, | ||
|
r34467 | urllibcompat, | ||
|
r25990 | util, | ||
) | ||||
|
r43346 | from .utils import stringutil | ||
|
r29455 | |||
httplib = util.httplib | ||||
|
r28861 | stringio = util.stringio | ||
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
|
r43346 | |||
|
r34695 | def escape(s, quote=None): | ||
'''Replace special characters "&", "<" and ">" to HTML-safe sequences. | ||||
If the optional flag quote is true, the quotation mark character (") | ||||
is also translated. | ||||
This is the same as cgi.escape in Python, but always operates on | ||||
bytes, whereas cgi.escape in Python 3 only works on unicodes. | ||||
''' | ||||
s = s.replace(b"&", b"&") | ||||
s = s.replace(b"<", b"<") | ||||
s = s.replace(b">", b">") | ||||
if quote: | ||||
s = s.replace(b'"', b""") | ||||
return s | ||||
|
r43346 | |||
|
r29377 | class passwordmgr(object): | ||
def __init__(self, ui, passwddb): | ||||
|
r7270 | self.ui = ui | ||
|
r29377 | self.passwddb = passwddb | ||
def add_password(self, realm, uri, user, passwd): | ||||
return self.passwddb.add_password(realm, uri, user, passwd) | ||||
|
r7270 | |||
def find_user_password(self, realm, authuri): | ||||
|
r41492 | assert isinstance(realm, (type(None), str)) | ||
assert isinstance(authuri, str) | ||||
|
r29377 | authinfo = self.passwddb.find_user_password(realm, authuri) | ||
|
r7270 | user, passwd = authinfo | ||
|
r41493 | user, passwd = pycompat.bytesurl(user), pycompat.bytesurl(passwd) | ||
|
r7270 | if user and passwd: | ||
|
r8333 | self._writedebug(user, passwd) | ||
|
r41730 | return (pycompat.strurl(user), pycompat.strurl(passwd)) | ||
|
r7270 | |||
|
r15005 | if not user or not passwd: | ||
|
r15025 | res = httpconnectionmod.readauthforuri(self.ui, authuri, user) | ||
|
r13372 | if res: | ||
group, auth = res | ||||
|
r43347 | user, passwd = auth.get(b'username'), auth.get(b'password') | ||
self.ui.debug(b"using auth.%s.* for authentication\n" % group) | ||||
|
r8333 | if not user or not passwd: | ||
|
r36670 | u = util.url(pycompat.bytesurl(authuri)) | ||
|
r20291 | u.query = None | ||
|
r8333 | if not self.ui.interactive(): | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'http authorization required for %s') | ||
|
r43346 | % util.hidepassword(bytes(u)) | ||
) | ||||
|
r7270 | |||
|
r43346 | self.ui.write( | ||
|
r43347 | _(b"http authorization required for %s\n") | ||
|
r43346 | % util.hidepassword(bytes(u)) | ||
) | ||||
|
r43347 | self.ui.write(_(b"realm: %s\n") % pycompat.bytesurl(realm)) | ||
|
r8333 | if user: | ||
|
r43347 | self.ui.write(_(b"user: %s\n") % user) | ||
|
r8333 | else: | ||
|
r43347 | user = self.ui.prompt(_(b"user:"), default=None) | ||
|
r7270 | |||
|
r8333 | if not passwd: | ||
passwd = self.ui.getpass() | ||||
|
r7270 | |||
|
r29377 | self.passwddb.add_password(realm, authuri, user, passwd) | ||
|
r8333 | self._writedebug(user, passwd) | ||
|
r41730 | return (pycompat.strurl(user), pycompat.strurl(passwd)) | ||
|
r7270 | |||
|
r8333 | def _writedebug(self, user, passwd): | ||
|
r43347 | msg = _(b'http auth: user %s, password %s\n') | ||
self.ui.debug(msg % (user, passwd and b'*' * len(passwd) or b'not set')) | ||||
|
r8333 | |||
|
r15025 | def find_stored_password(self, authuri): | ||
|
r29377 | return self.passwddb.find_user_password(None, authuri) | ||
|
r15025 | |||
|
r43346 | |||
|
r28883 | class proxyhandler(urlreq.proxyhandler): | ||
|
r7270 | def __init__(self, ui): | ||
|
r43347 | proxyurl = ui.config(b"http_proxy", b"host") or encoding.environ.get( | ||
b'http_proxy' | ||||
|
r43346 | ) | ||
|
r7270 | # XXX proxyauthinfo = None | ||
if proxyurl: | ||||
# proxy can be proper url or host[:port] | ||||
|
r43346 | if not ( | ||
|
r43347 | proxyurl.startswith(b'http:') or proxyurl.startswith(b'https:') | ||
|
r43346 | ): | ||
|
r43347 | proxyurl = b'http://' + proxyurl + b'/' | ||
|
r14076 | proxy = util.url(proxyurl) | ||
|
r13820 | if not proxy.user: | ||
|
r43347 | proxy.user = ui.config(b"http_proxy", b"user") | ||
proxy.passwd = ui.config(b"http_proxy", b"passwd") | ||||
|
r7270 | |||
# see if we should use a proxy for this url | ||||
|
r43347 | no_list = [b"localhost", b"127.0.0.1"] | ||
|
r43346 | no_list.extend( | ||
|
r43347 | [p.lower() for p in ui.configlist(b"http_proxy", b"no")] | ||
|
r43346 | ) | ||
no_list.extend( | ||||
[ | ||||
p.strip().lower() | ||||
|
r43347 | for p in encoding.environ.get(b"no_proxy", b'').split(b',') | ||
|
r43346 | if p.strip() | ||
] | ||||
) | ||||
|
r7270 | # "http_proxy.always" config is for running tests on localhost | ||
|
r43347 | if ui.configbool(b"http_proxy", b"always"): | ||
|
r7270 | self.no_list = [] | ||
else: | ||||
self.no_list = no_list | ||||
|
r41859 | # Keys and values need to be str because the standard library | ||
# expects them to be. | ||||
proxyurl = str(proxy) | ||||
proxies = {r'http': proxyurl, r'https': proxyurl} | ||||
|
r43347 | ui.debug(b'proxying through %s\n' % util.hidepassword(bytes(proxy))) | ||
|
r7270 | else: | ||
proxies = {} | ||||
|
r28883 | urlreq.proxyhandler.__init__(self, proxies) | ||
|
r7270 | self.ui = ui | ||
def proxy_open(self, req, proxy, type_): | ||||
|
r43347 | host = pycompat.bytesurl(urllibcompat.gethost(req)).split(b':')[0] | ||
|
r19535 | for e in self.no_list: | ||
if host == e: | ||||
return None | ||||
|
r43347 | if e.startswith(b'*.') and host.endswith(e[2:]): | ||
|
r19535 | return None | ||
|
r43347 | if e.startswith(b'.') and host.endswith(e[1:]): | ||
|
r19535 | return None | ||
|
r7270 | |||
|
r28883 | return urlreq.proxyhandler.proxy_open(self, req, proxy, type_) | ||
|
r7270 | |||
|
r43346 | |||
|
r13420 | def _gen_sendfile(orgsend): | ||
|
r7270 | def _sendfile(self, data): | ||
# send a file | ||||
|
r14244 | if isinstance(data, httpconnectionmod.httpsendfile): | ||
|
r7270 | # if auth required, some data sent twice, so rewind here | ||
data.seek(0) | ||||
for chunk in util.filechunkiter(data): | ||||
|
r13420 | orgsend(self, chunk) | ||
|
r7270 | else: | ||
|
r13420 | orgsend(self, data) | ||
|
r43346 | |||
|
r7270 | return _sendfile | ||
|
r43346 | |||
|
r43347 | has_https = util.safehasattr(urlreq, b'httpshandler') | ||
|
r10409 | |||
|
r43346 | |||
|
r7270 | class httpconnection(keepalive.HTTPConnection): | ||
# must be able to send big bundle as stream. | ||||
|
r13420 | send = _gen_sendfile(keepalive.HTTPConnection.send) | ||
|
r7270 | |||
|
r8590 | def getresponse(self): | ||
proxyres = getattr(self, 'proxyres', None) | ||||
if proxyres: | ||||
if proxyres.will_close: | ||||
self.close() | ||||
self.proxyres = None | ||||
return proxyres | ||||
return keepalive.HTTPConnection.getresponse(self) | ||||
|
r43346 | |||
|
r41857 | # Large parts of this function have their origin from before Python 2.6 | ||
# and could potentially be removed. | ||||
|
r9852 | def _generic_start_transaction(handler, h, req): | ||
|
r41857 | tunnel_host = req._tunnel_host | ||
|
r14964 | if tunnel_host: | ||
|
r41859 | if tunnel_host[:7] not in [r'http://', r'https:/']: | ||
tunnel_host = r'https://' + tunnel_host | ||||
|
r9852 | new_tunnel = True | ||
else: | ||||
|
r34467 | tunnel_host = urllibcompat.getselector(req) | ||
|
r9852 | new_tunnel = False | ||
|
r43346 | if new_tunnel or tunnel_host == urllibcompat.getfullurl(req): # has proxy | ||
|
r41859 | u = util.url(pycompat.bytesurl(tunnel_host)) | ||
|
r43347 | if new_tunnel or u.scheme == b'https': # only use CONNECT for HTTPS | ||
h.realhostport = b':'.join([u.host, (u.port or b'443')]) | ||||
|
r9852 | h.headers = req.headers.copy() | ||
h.headers.update(handler.parent.addheaders) | ||||
return | ||||
|
r10415 | h.realhostport = None | ||
|
r9852 | h.headers = None | ||
|
r43346 | |||
|
r9852 | def _generic_proxytunnel(self): | ||
proxyheaders = dict( | ||||
|
r43346 | [ | ||
(x, self.headers[x]) | ||||
for x in self.headers | ||||
if x.lower().startswith(r'proxy-') | ||||
] | ||||
) | ||||
|
r43347 | self.send(b'CONNECT %s HTTP/1.0\r\n' % self.realhostport) | ||
|
r43376 | for header in pycompat.iteritems(proxyheaders): | ||
|
r43347 | self.send(b'%s: %s\r\n' % header) | ||
self.send(b'\r\n') | ||||
|
r9852 | |||
# majority of the following code is duplicated from | ||||
# httplib.HTTPConnection as there are no adequate places to | ||||
# override functions to provide the needed functionality | ||||
|
r41860 | # strict was removed in Python 3.4. | ||
kwargs = {} | ||||
if not pycompat.ispy3: | ||||
|
r43347 | kwargs[b'strict'] = self.strict | ||
|
r41860 | |||
|
r43346 | res = self.response_class(self.sock, method=self._method, **kwargs) | ||
|
r9852 | |||
while True: | ||||
version, status, reason = res._read_status() | ||||
if status != httplib.CONTINUE: | ||||
break | ||||
|
r29729 | # skip lines that are all whitespace | ||
|
r43347 | list(iter(lambda: res.fp.readline().strip(), b'')) | ||
|
r9852 | res.status = status | ||
res.reason = reason.strip() | ||||
if res.status == 200: | ||||
|
r29729 | # skip lines until we find a blank line | ||
|
r43347 | list(iter(res.fp.readline, b'\r\n')) | ||
|
r9852 | return True | ||
|
r43347 | if version == b'HTTP/1.0': | ||
|
r9852 | res.version = 10 | ||
|
r43347 | elif version.startswith(b'HTTP/1.'): | ||
|
r9852 | res.version = 11 | ||
|
r43347 | elif version == b'HTTP/0.9': | ||
|
r9852 | res.version = 9 | ||
else: | ||||
raise httplib.UnknownProtocol(version) | ||||
if res.version == 9: | ||||
res.length = None | ||||
res.chunked = 0 | ||||
res.will_close = 1 | ||||
|
r28861 | res.msg = httplib.HTTPMessage(stringio()) | ||
|
r9852 | return False | ||
res.msg = httplib.HTTPMessage(res.fp) | ||||
res.msg.fp = None | ||||
# are we using the chunked-style of transfer encoding? | ||||
|
r43347 | trenc = res.msg.getheader(b'transfer-encoding') | ||
if trenc and trenc.lower() == b"chunked": | ||||
|
r9852 | res.chunked = 1 | ||
res.chunk_left = None | ||||
else: | ||||
res.chunked = 0 | ||||
# will the connection close at the end of the response? | ||||
res.will_close = res._check_close() | ||||
# do we have a Content-Length? | ||||
|
r17428 | # NOTE: RFC 2616, section 4.4, #3 says we ignore this if | ||
# transfer-encoding is "chunked" | ||||
|
r43347 | length = res.msg.getheader(b'content-length') | ||
|
r9852 | if length and not res.chunked: | ||
try: | ||||
res.length = int(length) | ||||
except ValueError: | ||||
res.length = None | ||||
else: | ||||
if res.length < 0: # ignore nonsensical negative lengths | ||||
res.length = None | ||||
else: | ||||
res.length = None | ||||
# does the body have a fixed length? (of zero) | ||||
|
r43346 | if ( | ||
status == httplib.NO_CONTENT | ||||
or status == httplib.NOT_MODIFIED | ||||
or 100 <= status < 200 | ||||
|
r43347 | or res._method == b'HEAD' # 1xx codes | ||
|
r43346 | ): | ||
|
r9852 | res.length = 0 | ||
# if the connection remains open, and we aren't using chunked, and | ||||
# a content-length was not provided, then assume that the connection | ||||
# WILL close. | ||||
|
r43346 | if not res.will_close and not res.chunked and res.length is None: | ||
|
r9852 | res.will_close = 1 | ||
self.proxyres = res | ||||
return False | ||||
|
r43346 | |||
|
r7270 | class httphandler(keepalive.HTTPHandler): | ||
def http_open(self, req): | ||||
return self.do_open(httpconnection, req) | ||||
|
r8590 | def _start_transaction(self, h, req): | ||
|
r9852 | _generic_start_transaction(self, h, req) | ||
|
r8590 | return keepalive.HTTPHandler._start_transaction(self, h, req) | ||
|
r43346 | |||
|
r37029 | class logginghttpconnection(keepalive.HTTPConnection): | ||
def __init__(self, createconn, *args, **kwargs): | ||||
keepalive.HTTPConnection.__init__(self, *args, **kwargs) | ||||
self._create_connection = createconn | ||||
|
r37118 | if sys.version_info < (2, 7, 7): | ||
# copied from 2.7.14, since old implementations directly call | ||||
# socket.create_connection() | ||||
def connect(self): | ||||
|
r43346 | self.sock = self._create_connection( | ||
(self.host, self.port), self.timeout, self.source_address | ||||
) | ||||
|
r37118 | if self._tunnel_host: | ||
self._tunnel() | ||||
|
r43346 | |||
|
r37029 | class logginghttphandler(httphandler): | ||
"""HTTP handler that logs socket I/O.""" | ||||
|
r43346 | |||
|
r40079 | def __init__(self, logfh, name, observeropts, timeout=None): | ||
super(logginghttphandler, self).__init__(timeout=timeout) | ||||
|
r37029 | |||
self._logfh = logfh | ||||
self._logname = name | ||||
self._observeropts = observeropts | ||||
# do_open() calls the passed class to instantiate an HTTPConnection. We | ||||
# pass in a callable method that creates a custom HTTPConnection instance | ||||
# whose callback to create the socket knows how to proxy the socket. | ||||
def http_open(self, req): | ||||
return self.do_open(self._makeconnection, req) | ||||
def _makeconnection(self, *args, **kwargs): | ||||
def createconnection(*args, **kwargs): | ||||
sock = socket.create_connection(*args, **kwargs) | ||||
|
r43346 | return util.makeloggingsocket( | ||
self._logfh, sock, self._logname, **self._observeropts | ||||
) | ||||
|
r37029 | |||
return logginghttpconnection(createconnection, *args, **kwargs) | ||||
|
r43346 | |||
|
r7270 | if has_https: | ||
|
r43346 | |||
|
r40067 | class httpsconnection(keepalive.HTTPConnection): | ||
|
r13424 | response_class = keepalive.HTTPResponse | ||
|
r25414 | default_port = httplib.HTTPS_PORT | ||
|
r13424 | # must be able to send big bundle as stream. | ||
send = _gen_sendfile(keepalive.safesend) | ||||
|
r25414 | getresponse = keepalive.wrapgetresponse(httplib.HTTPConnection) | ||
|
r43346 | def __init__( | ||
self, | ||||
host, | ||||
port=None, | ||||
key_file=None, | ||||
cert_file=None, | ||||
*args, | ||||
**kwargs | ||||
): | ||||
|
r40067 | keepalive.HTTPConnection.__init__(self, host, port, *args, **kwargs) | ||
|
r25414 | self.key_file = key_file | ||
self.cert_file = cert_file | ||||
|
r9726 | |||
|
r10409 | def connect(self): | ||
|
r29662 | self.sock = socket.create_connection((self.host, self.port)) | ||
|
r13422 | |||
|
r13421 | host = self.host | ||
|
r43346 | if self.realhostport: # use CONNECT proxy | ||
|
r14064 | _generic_proxytunnel(self) | ||
|
r43347 | host = self.realhostport.rsplit(b':', 1)[0] | ||
|
r25429 | self.sock = sslutil.wrapsocket( | ||
|
r43346 | self.sock, | ||
self.key_file, | ||||
self.cert_file, | ||||
ui=self.ui, | ||||
serverhostname=host, | ||||
) | ||||
|
r29227 | sslutil.validatesocket(self.sock) | ||
|
r10409 | |||
|
r28883 | class httpshandler(keepalive.KeepAliveHandler, urlreq.httpshandler): | ||
|
r40079 | def __init__(self, ui, timeout=None): | ||
keepalive.KeepAliveHandler.__init__(self, timeout=timeout) | ||||
|
r28883 | urlreq.httpshandler.__init__(self) | ||
|
r8847 | self.ui = ui | ||
|
r43346 | self.pwmgr = passwordmgr(self.ui, self.ui.httppasswordmgrdb) | ||
|
r8847 | |||
|
r9852 | def _start_transaction(self, h, req): | ||
_generic_start_transaction(self, h, req) | ||||
return keepalive.KeepAliveHandler._start_transaction(self, h, req) | ||||
|
r7270 | def https_open(self, req): | ||
|
r34467 | # urllibcompat.getfullurl() does not contain credentials | ||
# and we may need them to match the certificates. | ||||
url = urllibcompat.getfullurl(req) | ||||
|
r15025 | user, password = self.pwmgr.find_stored_password(url) | ||
res = httpconnectionmod.readauthforuri(self.ui, url, user) | ||||
|
r13372 | if res: | ||
group, auth = res | ||||
self.auth = auth | ||||
|
r43347 | self.ui.debug(b"using auth.%s.* for authentication\n" % group) | ||
|
r13372 | else: | ||
self.auth = None | ||||
|
r8847 | return self.do_open(self._makeconnection, req) | ||
|
r10408 | def _makeconnection(self, host, port=None, *args, **kwargs): | ||
|
r8847 | keyfile = None | ||
certfile = None | ||||
|
r43346 | if len(args) >= 1: # key_file | ||
|
r10511 | keyfile = args[0] | ||
|
r43346 | if len(args) >= 2: # cert_file | ||
|
r10511 | certfile = args[1] | ||
args = args[2:] | ||||
|
r8847 | |||
# if the user has specified different key/cert files in | ||||
# hgrc, we prefer these | ||||
|
r43347 | if self.auth and b'key' in self.auth and b'cert' in self.auth: | ||
keyfile = self.auth[b'key'] | ||||
certfile = self.auth[b'cert'] | ||||
|
r8847 | |||
|
r43346 | conn = httpsconnection( | ||
host, port, keyfile, certfile, *args, **kwargs | ||||
) | ||||
|
r10409 | conn.ui = self.ui | ||
return conn | ||||
|
r7270 | |||
|
r43346 | |||
|
r28883 | class httpdigestauthhandler(urlreq.httpdigestauthhandler): | ||
|
r11457 | def __init__(self, *args, **kwargs): | ||
|
r28883 | urlreq.httpdigestauthhandler.__init__(self, *args, **kwargs) | ||
|
r11457 | self.retried_req = None | ||
def reset_retry_count(self): | ||||
# Python 2.6.5 will call this on 401 or 407 errors and thus loop | ||||
# forever. We disable reset_retry_count completely and reset in | ||||
# http_error_auth_reqed instead. | ||||
pass | ||||
|
r7270 | def http_error_auth_reqed(self, auth_header, host, req, headers): | ||
|
r11457 | # Reset the retry counter once for each request. | ||
if req is not self.retried_req: | ||||
self.retried_req = req | ||||
self.retried = 0 | ||||
|
r28883 | return urlreq.httpdigestauthhandler.http_error_auth_reqed( | ||
|
r43346 | self, auth_header, host, req, headers | ||
) | ||||
|
r7270 | |||
|
r28883 | class httpbasicauthhandler(urlreq.httpbasicauthhandler): | ||
|
r11844 | def __init__(self, *args, **kwargs): | ||
|
r20964 | self.auth = None | ||
|
r28883 | urlreq.httpbasicauthhandler.__init__(self, *args, **kwargs) | ||
|
r11844 | self.retried_req = None | ||
|
r20964 | def http_request(self, request): | ||
if self.auth: | ||||
request.add_unredirected_header(self.auth_header, self.auth) | ||||
return request | ||||
def https_request(self, request): | ||||
if self.auth: | ||||
request.add_unredirected_header(self.auth_header, self.auth) | ||||
return request | ||||
|
r11844 | def reset_retry_count(self): | ||
# Python 2.6.5 will call this on 401 or 407 errors and thus loop | ||||
# forever. We disable reset_retry_count completely and reset in | ||||
# http_error_auth_reqed instead. | ||||
pass | ||||
def http_error_auth_reqed(self, auth_header, host, req, headers): | ||||
# Reset the retry counter once for each request. | ||||
if req is not self.retried_req: | ||||
self.retried_req = req | ||||
self.retried = 0 | ||||
|
r28883 | return urlreq.httpbasicauthhandler.http_error_auth_reqed( | ||
|
r43346 | self, auth_header, host, req, headers | ||
) | ||||
|
r11844 | |||
|
r20964 | def retry_http_basic_auth(self, host, req, realm): | ||
|
r34467 | user, pw = self.passwd.find_user_password( | ||
|
r43346 | realm, urllibcompat.getfullurl(req) | ||
) | ||||
|
r20964 | if pw is not None: | ||
|
r43347 | raw = b"%s:%s" % (pycompat.bytesurl(user), pycompat.bytesurl(pw)) | ||
|
r36670 | auth = r'Basic %s' % pycompat.strurl(base64.b64encode(raw).strip()) | ||
|
r29639 | if req.get_header(self.auth_header, None) == auth: | ||
|
r20964 | return None | ||
self.auth = auth | ||||
req.add_unredirected_header(self.auth_header, auth) | ||||
return self.parent.open(req) | ||||
else: | ||||
return None | ||||
|
r43346 | |||
|
r31936 | class cookiehandler(urlreq.basehandler): | ||
def __init__(self, ui): | ||||
self.cookiejar = None | ||||
|
r43347 | cookiefile = ui.config(b'auth', b'cookiefile') | ||
|
r31936 | if not cookiefile: | ||
return | ||||
cookiefile = util.expandpath(cookiefile) | ||||
try: | ||||
|
r37753 | cookiejar = util.cookielib.MozillaCookieJar( | ||
|
r43346 | pycompat.fsdecode(cookiefile) | ||
) | ||||
|
r31936 | cookiejar.load() | ||
self.cookiejar = cookiejar | ||||
except util.cookielib.LoadError as e: | ||||
|
r43346 | ui.warn( | ||
_( | ||||
|
r43347 | b'(error loading cookie file %s: %s; continuing without ' | ||
b'cookies)\n' | ||||
|
r43346 | ) | ||
% (cookiefile, stringutil.forcebytestr(e)) | ||||
) | ||||
|
r31936 | |||
def http_request(self, request): | ||||
if self.cookiejar: | ||||
self.cookiejar.add_cookie_header(request) | ||||
return request | ||||
def https_request(self, request): | ||||
if self.cookiejar: | ||||
self.cookiejar.add_cookie_header(request) | ||||
return request | ||||
|
r43346 | |||
|
r9347 | handlerfuncs = [] | ||
|
r43346 | |||
def opener( | ||||
ui, | ||||
authinfo=None, | ||||
useragent=None, | ||||
loggingfh=None, | ||||
loggingname=b's', | ||||
loggingopts=None, | ||||
sendaccept=True, | ||||
): | ||||
|
r7270 | ''' | ||
construct an opener suitable for urllib2 | ||||
authinfo will be added to the password manager | ||||
|
r37029 | |||
The opener can be configured to log socket events if the various | ||||
``logging*`` arguments are specified. | ||||
``loggingfh`` denotes a file object to log events to. | ||||
``loggingname`` denotes the name of the to print when logging. | ||||
``loggingopts`` is a dict of keyword arguments to pass to the constructed | ||||
``util.socketobserver`` instance. | ||||
|
r37063 | |||
``sendaccept`` allows controlling whether the ``Accept`` request header | ||||
is sent. The header is sent by default. | ||||
|
r7270 | ''' | ||
|
r43347 | timeout = ui.configwith(float, b'http', b'timeout') | ||
|
r37029 | handlers = [] | ||
if loggingfh: | ||||
|
r43346 | handlers.append( | ||
logginghttphandler( | ||||
loggingfh, loggingname, loggingopts or {}, timeout=timeout | ||||
) | ||||
) | ||||
|
r37029 | # We don't yet support HTTPS when logging I/O. If we attempt to open | ||
# an HTTPS URL, we'll likely fail due to unknown protocol. | ||||
else: | ||||
|
r40079 | handlers.append(httphandler(timeout=timeout)) | ||
|
r37029 | if has_https: | ||
|
r40079 | handlers.append(httpshandler(ui, timeout=timeout)) | ||
|
r7270 | |||
handlers.append(proxyhandler(ui)) | ||||
|
r29378 | passmgr = passwordmgr(ui, ui.httppasswordmgrdb) | ||
|
r7270 | if authinfo is not None: | ||
|
r29379 | realm, uris, user, passwd = authinfo | ||
saveduser, savedpass = passmgr.find_stored_password(uris[0]) | ||||
if user != saveduser or passwd: | ||||
passmgr.add_password(realm, uris, user, passwd) | ||||
|
r43346 | ui.debug( | ||
|
r43347 | b'http auth: user %s, password %s\n' | ||
% (user, passwd and b'*' * len(passwd) or b'not set') | ||||
|
r43346 | ) | ||
|
r7270 | |||
|
r43346 | handlers.extend( | ||
(httpbasicauthhandler(passmgr), httpdigestauthhandler(passmgr)) | ||||
) | ||||
|
r9347 | handlers.extend([h(ui, passmgr) for h in handlerfuncs]) | ||
|
r31936 | handlers.append(cookiehandler(ui)) | ||
|
r28883 | opener = urlreq.buildopener(*handlers) | ||
|
r7270 | |||
|
r40070 | # keepalive.py's handlers will populate these attributes if they exist. | ||
opener.requestscount = 0 | ||||
opener.sentbytescount = 0 | ||||
opener.receivedbytescount = 0 | ||||
|
r29589 | # The user agent should should *NOT* be used by servers for e.g. | ||
# protocol detection or feature negotiation: there are other | ||||
# facilities for that. | ||||
# | ||||
# "mercurial/proto-1.0" was the original user agent string and | ||||
# exists for backwards compatibility reasons. | ||||
# | ||||
# The "(Mercurial %s)" string contains the distribution | ||||
# name and version. Other client implementations should choose their | ||||
# own distribution name. Since servers should not be using the user | ||||
# agent string for anything, clients should be able to define whatever | ||||
# user agent they deem appropriate. | ||||
|
r35455 | # | ||
# The custom user agent is for lfs, because unfortunately some servers | ||||
# do look at this value. | ||||
if not useragent: | ||||
|
r43347 | agent = b'mercurial/proto-1.0 (Mercurial %s)' % util.version() | ||
|
r35455 | opener.addheaders = [(r'User-agent', pycompat.sysstr(agent))] | ||
else: | ||||
opener.addheaders = [(r'User-agent', pycompat.sysstr(useragent))] | ||||
|
r30763 | |||
# This header should only be needed by wire protocol requests. But it has | ||||
# been sent on all requests since forever. We keep sending it for backwards | ||||
# compatibility reasons. Modern versions of the wire protocol use | ||||
# X-HgProto-<N> for advertising client support. | ||||
|
r37063 | if sendaccept: | ||
opener.addheaders.append((r'Accept', r'application/mercurial-0.1')) | ||||
|
r7270 | return opener | ||
|
r43346 | |||
|
r42278 | def open(ui, url_, data=None, sendaccept=True): | ||
|
r14076 | u = util.url(url_) | ||
|
r13818 | if u.scheme: | ||
u.scheme = u.scheme.lower() | ||||
url_, authinfo = u.authinfo() | ||||
else: | ||||
path = util.normpath(os.path.abspath(url_)) | ||||
|
r43347 | url_ = b'file://' + pycompat.bytesurl(urlreq.pathname2url(path)) | ||
|
r7284 | authinfo = None | ||
|
r43346 | return opener(ui, authinfo, sendaccept=sendaccept).open( | ||
pycompat.strurl(url_), data | ||||
) | ||||
|
r40054 | |||
def wrapresponse(resp): | ||||
"""Wrap a response object with common error handlers. | ||||
This ensures that any I/O from any consumer raises the appropriate | ||||
error and messaging. | ||||
""" | ||||
origread = resp.read | ||||
class readerproxy(resp.__class__): | ||||
def read(self, size=None): | ||||
try: | ||||
return origread(size) | ||||
except httplib.IncompleteRead as e: | ||||
# e.expected is an integer if length known or None otherwise. | ||||
if e.expected: | ||||
got = len(e.partial) | ||||
total = e.expected + got | ||||
|
r43346 | msg = _( | ||
|
r43347 | b'HTTP request error (incomplete response; ' | ||
b'expected %d bytes got %d)' | ||||
|
r43346 | ) % (total, got) | ||
|
r40054 | else: | ||
|
r43347 | msg = _(b'HTTP request error (incomplete response)') | ||
|
r40054 | |||
raise error.PeerTransportError( | ||||
msg, | ||||
|
r43346 | hint=_( | ||
|
r43347 | b'this may be an intermittent network failure; ' | ||
b'if the error persists, consider contacting the ' | ||||
b'network or server operator' | ||||
|
r43346 | ), | ||
) | ||||
|
r40054 | except httplib.HTTPException as e: | ||
raise error.PeerTransportError( | ||||
|
r43347 | _(b'HTTP request error (%s)') % e, | ||
|
r43346 | hint=_( | ||
|
r43347 | b'this may be an intermittent network failure; ' | ||
b'if the error persists, consider contacting the ' | ||||
b'network or server operator' | ||||
|
r43346 | ), | ||
) | ||||
|
r40054 | |||
resp.__class__ = readerproxy | ||||