##// END OF EJS Templates
httpconnection: remove use of sslkwargs...
Gregory Szorc -
r29250:d6b9468e default
parent child Browse files
Show More
@@ -1,290 +1,288 b''
1 # httpconnection.py - urllib2 handler for new http support
1 # httpconnection.py - urllib2 handler for new http support
2 #
2 #
3 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>
4 # Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>
4 # Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 # Copyright 2011 Google, Inc.
6 # Copyright 2011 Google, Inc.
7 #
7 #
8 # This software may be used and distributed according to the terms of the
8 # This software may be used and distributed according to the terms of the
9 # GNU General Public License version 2 or any later version.
9 # GNU General Public License version 2 or any later version.
10
10
11 from __future__ import absolute_import
11 from __future__ import absolute_import
12
12
13 import logging
13 import logging
14 import os
14 import os
15 import socket
15 import socket
16
16
17 from .i18n import _
17 from .i18n import _
18 from . import (
18 from . import (
19 httpclient,
19 httpclient,
20 sslutil,
20 sslutil,
21 util,
21 util,
22 )
22 )
23
23
24 urlerr = util.urlerr
24 urlerr = util.urlerr
25 urlreq = util.urlreq
25 urlreq = util.urlreq
26
26
27 # moved here from url.py to avoid a cycle
27 # moved here from url.py to avoid a cycle
28 class httpsendfile(object):
28 class httpsendfile(object):
29 """This is a wrapper around the objects returned by python's "open".
29 """This is a wrapper around the objects returned by python's "open".
30
30
31 Its purpose is to send file-like objects via HTTP.
31 Its purpose is to send file-like objects via HTTP.
32 It do however not define a __len__ attribute because the length
32 It do however not define a __len__ attribute because the length
33 might be more than Py_ssize_t can handle.
33 might be more than Py_ssize_t can handle.
34 """
34 """
35
35
36 def __init__(self, ui, *args, **kwargs):
36 def __init__(self, ui, *args, **kwargs):
37 self.ui = ui
37 self.ui = ui
38 self._data = open(*args, **kwargs)
38 self._data = open(*args, **kwargs)
39 self.seek = self._data.seek
39 self.seek = self._data.seek
40 self.close = self._data.close
40 self.close = self._data.close
41 self.write = self._data.write
41 self.write = self._data.write
42 self.length = os.fstat(self._data.fileno()).st_size
42 self.length = os.fstat(self._data.fileno()).st_size
43 self._pos = 0
43 self._pos = 0
44 self._total = self.length // 1024 * 2
44 self._total = self.length // 1024 * 2
45
45
46 def read(self, *args, **kwargs):
46 def read(self, *args, **kwargs):
47 try:
47 try:
48 ret = self._data.read(*args, **kwargs)
48 ret = self._data.read(*args, **kwargs)
49 except EOFError:
49 except EOFError:
50 self.ui.progress(_('sending'), None)
50 self.ui.progress(_('sending'), None)
51 self._pos += len(ret)
51 self._pos += len(ret)
52 # We pass double the max for total because we currently have
52 # We pass double the max for total because we currently have
53 # to send the bundle twice in the case of a server that
53 # to send the bundle twice in the case of a server that
54 # requires authentication. Since we can't know until we try
54 # requires authentication. Since we can't know until we try
55 # once whether authentication will be required, just lie to
55 # once whether authentication will be required, just lie to
56 # the user and maybe the push succeeds suddenly at 50%.
56 # the user and maybe the push succeeds suddenly at 50%.
57 self.ui.progress(_('sending'), self._pos // 1024,
57 self.ui.progress(_('sending'), self._pos // 1024,
58 unit=_('kb'), total=self._total)
58 unit=_('kb'), total=self._total)
59 return ret
59 return ret
60
60
61 # moved here from url.py to avoid a cycle
61 # moved here from url.py to avoid a cycle
62 def readauthforuri(ui, uri, user):
62 def readauthforuri(ui, uri, user):
63 # Read configuration
63 # Read configuration
64 config = dict()
64 config = dict()
65 for key, val in ui.configitems('auth'):
65 for key, val in ui.configitems('auth'):
66 if '.' not in key:
66 if '.' not in key:
67 ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)
67 ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)
68 continue
68 continue
69 group, setting = key.rsplit('.', 1)
69 group, setting = key.rsplit('.', 1)
70 gdict = config.setdefault(group, dict())
70 gdict = config.setdefault(group, dict())
71 if setting in ('username', 'cert', 'key'):
71 if setting in ('username', 'cert', 'key'):
72 val = util.expandpath(val)
72 val = util.expandpath(val)
73 gdict[setting] = val
73 gdict[setting] = val
74
74
75 # Find the best match
75 # Find the best match
76 scheme, hostpath = uri.split('://', 1)
76 scheme, hostpath = uri.split('://', 1)
77 bestuser = None
77 bestuser = None
78 bestlen = 0
78 bestlen = 0
79 bestauth = None
79 bestauth = None
80 for group, auth in config.iteritems():
80 for group, auth in config.iteritems():
81 if user and user != auth.get('username', user):
81 if user and user != auth.get('username', user):
82 # If a username was set in the URI, the entry username
82 # If a username was set in the URI, the entry username
83 # must either match it or be unset
83 # must either match it or be unset
84 continue
84 continue
85 prefix = auth.get('prefix')
85 prefix = auth.get('prefix')
86 if not prefix:
86 if not prefix:
87 continue
87 continue
88 p = prefix.split('://', 1)
88 p = prefix.split('://', 1)
89 if len(p) > 1:
89 if len(p) > 1:
90 schemes, prefix = [p[0]], p[1]
90 schemes, prefix = [p[0]], p[1]
91 else:
91 else:
92 schemes = (auth.get('schemes') or 'https').split()
92 schemes = (auth.get('schemes') or 'https').split()
93 if (prefix == '*' or hostpath.startswith(prefix)) and \
93 if (prefix == '*' or hostpath.startswith(prefix)) and \
94 (len(prefix) > bestlen or (len(prefix) == bestlen and \
94 (len(prefix) > bestlen or (len(prefix) == bestlen and \
95 not bestuser and 'username' in auth)) \
95 not bestuser and 'username' in auth)) \
96 and scheme in schemes:
96 and scheme in schemes:
97 bestlen = len(prefix)
97 bestlen = len(prefix)
98 bestauth = group, auth
98 bestauth = group, auth
99 bestuser = auth.get('username')
99 bestuser = auth.get('username')
100 if user and not bestuser:
100 if user and not bestuser:
101 auth['username'] = user
101 auth['username'] = user
102 return bestauth
102 return bestauth
103
103
104 # Mercurial (at least until we can remove the old codepath) requires
104 # Mercurial (at least until we can remove the old codepath) requires
105 # that the http response object be sufficiently file-like, so we
105 # that the http response object be sufficiently file-like, so we
106 # provide a close() method here.
106 # provide a close() method here.
107 class HTTPResponse(httpclient.HTTPResponse):
107 class HTTPResponse(httpclient.HTTPResponse):
108 def close(self):
108 def close(self):
109 pass
109 pass
110
110
111 class HTTPConnection(httpclient.HTTPConnection):
111 class HTTPConnection(httpclient.HTTPConnection):
112 response_class = HTTPResponse
112 response_class = HTTPResponse
113 def request(self, method, uri, body=None, headers=None):
113 def request(self, method, uri, body=None, headers=None):
114 if headers is None:
114 if headers is None:
115 headers = {}
115 headers = {}
116 if isinstance(body, httpsendfile):
116 if isinstance(body, httpsendfile):
117 body.seek(0)
117 body.seek(0)
118 httpclient.HTTPConnection.request(self, method, uri, body=body,
118 httpclient.HTTPConnection.request(self, method, uri, body=body,
119 headers=headers)
119 headers=headers)
120
120
121
121
122 _configuredlogging = False
122 _configuredlogging = False
123 LOGFMT = '%(levelname)s:%(name)s:%(lineno)d:%(message)s'
123 LOGFMT = '%(levelname)s:%(name)s:%(lineno)d:%(message)s'
124 # Subclass BOTH of these because otherwise urllib2 "helpfully"
124 # Subclass BOTH of these because otherwise urllib2 "helpfully"
125 # reinserts them since it notices we don't include any subclasses of
125 # reinserts them since it notices we don't include any subclasses of
126 # them.
126 # them.
127 class http2handler(urlreq.httphandler, urlreq.httpshandler):
127 class http2handler(urlreq.httphandler, urlreq.httpshandler):
128 def __init__(self, ui, pwmgr):
128 def __init__(self, ui, pwmgr):
129 global _configuredlogging
129 global _configuredlogging
130 urlreq.abstracthttphandler.__init__(self)
130 urlreq.abstracthttphandler.__init__(self)
131 self.ui = ui
131 self.ui = ui
132 self.pwmgr = pwmgr
132 self.pwmgr = pwmgr
133 self._connections = {}
133 self._connections = {}
134 # developer config: ui.http2debuglevel
134 # developer config: ui.http2debuglevel
135 loglevel = ui.config('ui', 'http2debuglevel', default=None)
135 loglevel = ui.config('ui', 'http2debuglevel', default=None)
136 if loglevel and not _configuredlogging:
136 if loglevel and not _configuredlogging:
137 _configuredlogging = True
137 _configuredlogging = True
138 logger = logging.getLogger('mercurial.httpclient')
138 logger = logging.getLogger('mercurial.httpclient')
139 logger.setLevel(getattr(logging, loglevel.upper()))
139 logger.setLevel(getattr(logging, loglevel.upper()))
140 handler = logging.StreamHandler()
140 handler = logging.StreamHandler()
141 handler.setFormatter(logging.Formatter(LOGFMT))
141 handler.setFormatter(logging.Formatter(LOGFMT))
142 logger.addHandler(handler)
142 logger.addHandler(handler)
143
143
144 def close_all(self):
144 def close_all(self):
145 """Close and remove all connection objects being kept for reuse."""
145 """Close and remove all connection objects being kept for reuse."""
146 for openconns in self._connections.values():
146 for openconns in self._connections.values():
147 for conn in openconns:
147 for conn in openconns:
148 conn.close()
148 conn.close()
149 self._connections = {}
149 self._connections = {}
150
150
151 # shamelessly borrowed from urllib2.AbstractHTTPHandler
151 # shamelessly borrowed from urllib2.AbstractHTTPHandler
152 def do_open(self, http_class, req, use_ssl):
152 def do_open(self, http_class, req, use_ssl):
153 """Return an addinfourl object for the request, using http_class.
153 """Return an addinfourl object for the request, using http_class.
154
154
155 http_class must implement the HTTPConnection API from httplib.
155 http_class must implement the HTTPConnection API from httplib.
156 The addinfourl return value is a file-like object. It also
156 The addinfourl return value is a file-like object. It also
157 has methods and attributes including:
157 has methods and attributes including:
158 - info(): return a mimetools.Message object for the headers
158 - info(): return a mimetools.Message object for the headers
159 - geturl(): return the original request URL
159 - geturl(): return the original request URL
160 - code: HTTP status code
160 - code: HTTP status code
161 """
161 """
162 # If using a proxy, the host returned by get_host() is
162 # If using a proxy, the host returned by get_host() is
163 # actually the proxy. On Python 2.6.1, the real destination
163 # actually the proxy. On Python 2.6.1, the real destination
164 # hostname is encoded in the URI in the urllib2 request
164 # hostname is encoded in the URI in the urllib2 request
165 # object. On Python 2.6.5, it's stored in the _tunnel_host
165 # object. On Python 2.6.5, it's stored in the _tunnel_host
166 # attribute which has no accessor.
166 # attribute which has no accessor.
167 tunhost = getattr(req, '_tunnel_host', None)
167 tunhost = getattr(req, '_tunnel_host', None)
168 host = req.get_host()
168 host = req.get_host()
169 if tunhost:
169 if tunhost:
170 proxyhost = host
170 proxyhost = host
171 host = tunhost
171 host = tunhost
172 elif req.has_proxy():
172 elif req.has_proxy():
173 proxyhost = req.get_host()
173 proxyhost = req.get_host()
174 host = req.get_selector().split('://', 1)[1].split('/', 1)[0]
174 host = req.get_selector().split('://', 1)[1].split('/', 1)[0]
175 else:
175 else:
176 proxyhost = None
176 proxyhost = None
177
177
178 if proxyhost:
178 if proxyhost:
179 if ':' in proxyhost:
179 if ':' in proxyhost:
180 # Note: this means we'll explode if we try and use an
180 # Note: this means we'll explode if we try and use an
181 # IPv6 http proxy. This isn't a regression, so we
181 # IPv6 http proxy. This isn't a regression, so we
182 # won't worry about it for now.
182 # won't worry about it for now.
183 proxyhost, proxyport = proxyhost.rsplit(':', 1)
183 proxyhost, proxyport = proxyhost.rsplit(':', 1)
184 else:
184 else:
185 proxyport = 3128 # squid default
185 proxyport = 3128 # squid default
186 proxy = (proxyhost, proxyport)
186 proxy = (proxyhost, proxyport)
187 else:
187 else:
188 proxy = None
188 proxy = None
189
189
190 if not host:
190 if not host:
191 raise urlerr.urlerror('no host given')
191 raise urlerr.urlerror('no host given')
192
192
193 connkey = use_ssl, host, proxy
193 connkey = use_ssl, host, proxy
194 allconns = self._connections.get(connkey, [])
194 allconns = self._connections.get(connkey, [])
195 conns = [c for c in allconns if not c.busy()]
195 conns = [c for c in allconns if not c.busy()]
196 if conns:
196 if conns:
197 h = conns[0]
197 h = conns[0]
198 else:
198 else:
199 if allconns:
199 if allconns:
200 self.ui.debug('all connections for %s busy, making a new '
200 self.ui.debug('all connections for %s busy, making a new '
201 'one\n' % host)
201 'one\n' % host)
202 timeout = None
202 timeout = None
203 if req.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
203 if req.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
204 timeout = req.timeout
204 timeout = req.timeout
205 h = http_class(host, timeout=timeout, proxy_hostport=proxy)
205 h = http_class(host, timeout=timeout, proxy_hostport=proxy)
206 self._connections.setdefault(connkey, []).append(h)
206 self._connections.setdefault(connkey, []).append(h)
207
207
208 headers = dict(req.headers)
208 headers = dict(req.headers)
209 headers.update(req.unredirected_hdrs)
209 headers.update(req.unredirected_hdrs)
210 headers = dict(
210 headers = dict(
211 (name.title(), val) for name, val in headers.items())
211 (name.title(), val) for name, val in headers.items())
212 try:
212 try:
213 path = req.get_selector()
213 path = req.get_selector()
214 if '://' in path:
214 if '://' in path:
215 path = path.split('://', 1)[1].split('/', 1)[1]
215 path = path.split('://', 1)[1].split('/', 1)[1]
216 if path[0] != '/':
216 if path[0] != '/':
217 path = '/' + path
217 path = '/' + path
218 h.request(req.get_method(), path, req.data, headers)
218 h.request(req.get_method(), path, req.data, headers)
219 r = h.getresponse()
219 r = h.getresponse()
220 except socket.error as err: # XXX what error?
220 except socket.error as err: # XXX what error?
221 raise urlerr.urlerror(err)
221 raise urlerr.urlerror(err)
222
222
223 # Pick apart the HTTPResponse object to get the addinfourl
223 # Pick apart the HTTPResponse object to get the addinfourl
224 # object initialized properly.
224 # object initialized properly.
225 r.recv = r.read
225 r.recv = r.read
226
226
227 resp = urlreq.addinfourl(r, r.headers, req.get_full_url())
227 resp = urlreq.addinfourl(r, r.headers, req.get_full_url())
228 resp.code = r.status
228 resp.code = r.status
229 resp.msg = r.reason
229 resp.msg = r.reason
230 return resp
230 return resp
231
231
232 # httplib always uses the given host/port as the socket connect
232 # httplib always uses the given host/port as the socket connect
233 # target, and then allows full URIs in the request path, which it
233 # target, and then allows full URIs in the request path, which it
234 # then observes and treats as a signal to do proxying instead.
234 # then observes and treats as a signal to do proxying instead.
235 def http_open(self, req):
235 def http_open(self, req):
236 if req.get_full_url().startswith('https'):
236 if req.get_full_url().startswith('https'):
237 return self.https_open(req)
237 return self.https_open(req)
238 def makehttpcon(*args, **kwargs):
238 def makehttpcon(*args, **kwargs):
239 k2 = dict(kwargs)
239 k2 = dict(kwargs)
240 k2['use_ssl'] = False
240 k2['use_ssl'] = False
241 return HTTPConnection(*args, **k2)
241 return HTTPConnection(*args, **k2)
242 return self.do_open(makehttpcon, req, False)
242 return self.do_open(makehttpcon, req, False)
243
243
244 def https_open(self, req):
244 def https_open(self, req):
245 # req.get_full_url() does not contain credentials and we may
245 # req.get_full_url() does not contain credentials and we may
246 # need them to match the certificates.
246 # need them to match the certificates.
247 url = req.get_full_url()
247 url = req.get_full_url()
248 user, password = self.pwmgr.find_stored_password(url)
248 user, password = self.pwmgr.find_stored_password(url)
249 res = readauthforuri(self.ui, url, user)
249 res = readauthforuri(self.ui, url, user)
250 if res:
250 if res:
251 group, auth = res
251 group, auth = res
252 self.auth = auth
252 self.auth = auth
253 self.ui.debug("using auth.%s.* for authentication\n" % group)
253 self.ui.debug("using auth.%s.* for authentication\n" % group)
254 else:
254 else:
255 self.auth = None
255 self.auth = None
256 return self.do_open(self._makesslconnection, req, True)
256 return self.do_open(self._makesslconnection, req, True)
257
257
258 def _makesslconnection(self, host, port=443, *args, **kwargs):
258 def _makesslconnection(self, host, port=443, *args, **kwargs):
259 keyfile = None
259 keyfile = None
260 certfile = None
260 certfile = None
261
261
262 if args: # key_file
262 if args: # key_file
263 keyfile = args.pop(0)
263 keyfile = args.pop(0)
264 if args: # cert_file
264 if args: # cert_file
265 certfile = args.pop(0)
265 certfile = args.pop(0)
266
266
267 # if the user has specified different key/cert files in
267 # if the user has specified different key/cert files in
268 # hgrc, we prefer these
268 # hgrc, we prefer these
269 if self.auth and 'key' in self.auth and 'cert' in self.auth:
269 if self.auth and 'key' in self.auth and 'cert' in self.auth:
270 keyfile = self.auth['key']
270 keyfile = self.auth['key']
271 certfile = self.auth['cert']
271 certfile = self.auth['cert']
272
272
273 # let host port take precedence
273 # let host port take precedence
274 if ':' in host and '[' not in host or ']:' in host:
274 if ':' in host and '[' not in host or ']:' in host:
275 host, port = host.rsplit(':', 1)
275 host, port = host.rsplit(':', 1)
276 port = int(port)
276 port = int(port)
277 if '[' in host:
277 if '[' in host:
278 host = host[1:-1]
278 host = host[1:-1]
279
279
280 kwargs['keyfile'] = keyfile
280 kwargs['keyfile'] = keyfile
281 kwargs['certfile'] = certfile
281 kwargs['certfile'] = certfile
282
282
283 kwargs.update(sslutil.sslkwargs(self.ui, host))
284
285 con = HTTPConnection(host, port, use_ssl=True,
283 con = HTTPConnection(host, port, use_ssl=True,
286 ssl_wrap_socket=sslutil.wrapsocket,
284 ssl_wrap_socket=sslutil.wrapsocket,
287 ssl_validator=sslutil.validatesocket,
285 ssl_validator=sslutil.validatesocket,
288 ui=self.ui,
286 ui=self.ui,
289 **kwargs)
287 **kwargs)
290 return con
288 return con
General Comments 0
You need to be logged in to leave comments. Login now