##// END OF EJS Templates
httpclient: fix calling convention violation
Matt Mackall -
r17836:98347af6 default
parent child Browse files
Show More
@@ -1,286 +1,288 b''
1 # httpconnection.py - urllib2 handler for new http support
1 # httpconnection.py - urllib2 handler for new http support
2 #
2 #
3 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <mpm@selenic.com>
4 # Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>
4 # Copyright 2006, 2007 Alexis S. L. Carvalho <alexis@cecm.usp.br>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
6 # Copyright 2011 Google, Inc.
6 # Copyright 2011 Google, Inc.
7 #
7 #
8 # This software may be used and distributed according to the terms of the
8 # This software may be used and distributed according to the terms of the
9 # GNU General Public License version 2 or any later version.
9 # GNU General Public License version 2 or any later version.
10 import logging
10 import logging
11 import socket
11 import socket
12 import urllib
12 import urllib
13 import urllib2
13 import urllib2
14 import os
14 import os
15
15
16 from mercurial import httpclient
16 from mercurial import httpclient
17 from mercurial import sslutil
17 from mercurial import sslutil
18 from mercurial import util
18 from mercurial import util
19 from mercurial.i18n import _
19 from mercurial.i18n import _
20
20
21 # moved here from url.py to avoid a cycle
21 # moved here from url.py to avoid a cycle
22 class httpsendfile(object):
22 class httpsendfile(object):
23 """This is a wrapper around the objects returned by python's "open".
23 """This is a wrapper around the objects returned by python's "open".
24
24
25 Its purpose is to send file-like objects via HTTP.
25 Its purpose is to send file-like objects via HTTP.
26 It do however not define a __len__ attribute because the length
26 It do however not define a __len__ attribute because the length
27 might be more than Py_ssize_t can handle.
27 might be more than Py_ssize_t can handle.
28 """
28 """
29
29
30 def __init__(self, ui, *args, **kwargs):
30 def __init__(self, ui, *args, **kwargs):
31 # We can't just "self._data = open(*args, **kwargs)" here because there
31 # We can't just "self._data = open(*args, **kwargs)" here because there
32 # is an "open" function defined in this module that shadows the global
32 # is an "open" function defined in this module that shadows the global
33 # one
33 # one
34 self.ui = ui
34 self.ui = ui
35 self._data = open(*args, **kwargs)
35 self._data = open(*args, **kwargs)
36 self.seek = self._data.seek
36 self.seek = self._data.seek
37 self.close = self._data.close
37 self.close = self._data.close
38 self.write = self._data.write
38 self.write = self._data.write
39 self.length = os.fstat(self._data.fileno()).st_size
39 self.length = os.fstat(self._data.fileno()).st_size
40 self._pos = 0
40 self._pos = 0
41 self._total = self.length // 1024 * 2
41 self._total = self.length // 1024 * 2
42
42
43 def read(self, *args, **kwargs):
43 def read(self, *args, **kwargs):
44 try:
44 try:
45 ret = self._data.read(*args, **kwargs)
45 ret = self._data.read(*args, **kwargs)
46 except EOFError:
46 except EOFError:
47 self.ui.progress(_('sending'), None)
47 self.ui.progress(_('sending'), None)
48 self._pos += len(ret)
48 self._pos += len(ret)
49 # We pass double the max for total because we currently have
49 # We pass double the max for total because we currently have
50 # to send the bundle twice in the case of a server that
50 # to send the bundle twice in the case of a server that
51 # requires authentication. Since we can't know until we try
51 # requires authentication. Since we can't know until we try
52 # once whether authentication will be required, just lie to
52 # once whether authentication will be required, just lie to
53 # the user and maybe the push succeeds suddenly at 50%.
53 # the user and maybe the push succeeds suddenly at 50%.
54 self.ui.progress(_('sending'), self._pos // 1024,
54 self.ui.progress(_('sending'), self._pos // 1024,
55 unit=_('kb'), total=self._total)
55 unit=_('kb'), total=self._total)
56 return ret
56 return ret
57
57
58 # moved here from url.py to avoid a cycle
58 # moved here from url.py to avoid a cycle
59 def readauthforuri(ui, uri, user):
59 def readauthforuri(ui, uri, user):
60 # Read configuration
60 # Read configuration
61 config = dict()
61 config = dict()
62 for key, val in ui.configitems('auth'):
62 for key, val in ui.configitems('auth'):
63 if '.' not in key:
63 if '.' not in key:
64 ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)
64 ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)
65 continue
65 continue
66 group, setting = key.rsplit('.', 1)
66 group, setting = key.rsplit('.', 1)
67 gdict = config.setdefault(group, dict())
67 gdict = config.setdefault(group, dict())
68 if setting in ('username', 'cert', 'key'):
68 if setting in ('username', 'cert', 'key'):
69 val = util.expandpath(val)
69 val = util.expandpath(val)
70 gdict[setting] = val
70 gdict[setting] = val
71
71
72 # Find the best match
72 # Find the best match
73 if '://' in uri:
73 if '://' in uri:
74 scheme, hostpath = uri.split('://', 1)
74 scheme, hostpath = uri.split('://', 1)
75 else:
75 else:
76 # Python 2.4.1 doesn't provide the full URI
76 # Python 2.4.1 doesn't provide the full URI
77 scheme, hostpath = 'http', uri
77 scheme, hostpath = 'http', uri
78 bestuser = None
78 bestuser = None
79 bestlen = 0
79 bestlen = 0
80 bestauth = None
80 bestauth = None
81 for group, auth in config.iteritems():
81 for group, auth in config.iteritems():
82 if user and user != auth.get('username', user):
82 if user and user != auth.get('username', user):
83 # If a username was set in the URI, the entry username
83 # If a username was set in the URI, the entry username
84 # must either match it or be unset
84 # must either match it or be unset
85 continue
85 continue
86 prefix = auth.get('prefix')
86 prefix = auth.get('prefix')
87 if not prefix:
87 if not prefix:
88 continue
88 continue
89 p = prefix.split('://', 1)
89 p = prefix.split('://', 1)
90 if len(p) > 1:
90 if len(p) > 1:
91 schemes, prefix = [p[0]], p[1]
91 schemes, prefix = [p[0]], p[1]
92 else:
92 else:
93 schemes = (auth.get('schemes') or 'https').split()
93 schemes = (auth.get('schemes') or 'https').split()
94 if (prefix == '*' or hostpath.startswith(prefix)) and \
94 if (prefix == '*' or hostpath.startswith(prefix)) and \
95 (len(prefix) > bestlen or (len(prefix) == bestlen and \
95 (len(prefix) > bestlen or (len(prefix) == bestlen and \
96 not bestuser and 'username' in auth)) \
96 not bestuser and 'username' in auth)) \
97 and scheme in schemes:
97 and scheme in schemes:
98 bestlen = len(prefix)
98 bestlen = len(prefix)
99 bestauth = group, auth
99 bestauth = group, auth
100 bestuser = auth.get('username')
100 bestuser = auth.get('username')
101 if user and not bestuser:
101 if user and not bestuser:
102 auth['username'] = user
102 auth['username'] = user
103 return bestauth
103 return bestauth
104
104
105 # Mercurial (at least until we can remove the old codepath) requires
105 # Mercurial (at least until we can remove the old codepath) requires
106 # that the http response object be sufficiently file-like, so we
106 # that the http response object be sufficiently file-like, so we
107 # provide a close() method here.
107 # provide a close() method here.
108 class HTTPResponse(httpclient.HTTPResponse):
108 class HTTPResponse(httpclient.HTTPResponse):
109 def close(self):
109 def close(self):
110 pass
110 pass
111
111
112 class HTTPConnection(httpclient.HTTPConnection):
112 class HTTPConnection(httpclient.HTTPConnection):
113 response_class = HTTPResponse
113 response_class = HTTPResponse
114 def request(self, method, uri, body=None, headers={}):
114 def request(self, method, uri, body=None, headers={}):
115 if isinstance(body, httpsendfile):
115 if isinstance(body, httpsendfile):
116 body.seek(0)
116 body.seek(0)
117 httpclient.HTTPConnection.request(self, method, uri, body=body,
117 httpclient.HTTPConnection.request(self, method, uri, body=body,
118 headers=headers)
118 headers=headers)
119
119
120
120
121 _configuredlogging = False
121 _configuredlogging = False
122 LOGFMT = '%(levelname)s:%(name)s:%(lineno)d:%(message)s'
122 LOGFMT = '%(levelname)s:%(name)s:%(lineno)d:%(message)s'
123 # Subclass BOTH of these because otherwise urllib2 "helpfully"
123 # Subclass BOTH of these because otherwise urllib2 "helpfully"
124 # reinserts them since it notices we don't include any subclasses of
124 # reinserts them since it notices we don't include any subclasses of
125 # them.
125 # them.
126 class http2handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
126 class http2handler(urllib2.HTTPHandler, urllib2.HTTPSHandler):
127 def __init__(self, ui, pwmgr):
127 def __init__(self, ui, pwmgr):
128 global _configuredlogging
128 global _configuredlogging
129 urllib2.AbstractHTTPHandler.__init__(self)
129 urllib2.AbstractHTTPHandler.__init__(self)
130 self.ui = ui
130 self.ui = ui
131 self.pwmgr = pwmgr
131 self.pwmgr = pwmgr
132 self._connections = {}
132 self._connections = {}
133 loglevel = ui.config('ui', 'http2debuglevel', default=None)
133 loglevel = ui.config('ui', 'http2debuglevel', default=None)
134 if loglevel and not _configuredlogging:
134 if loglevel and not _configuredlogging:
135 _configuredlogging = True
135 _configuredlogging = True
136 logger = logging.getLogger('mercurial.httpclient')
136 logger = logging.getLogger('mercurial.httpclient')
137 logger.setLevel(getattr(logging, loglevel.upper()))
137 logger.setLevel(getattr(logging, loglevel.upper()))
138 handler = logging.StreamHandler()
138 handler = logging.StreamHandler()
139 handler.setFormatter(logging.Formatter(LOGFMT))
139 handler.setFormatter(logging.Formatter(LOGFMT))
140 logger.addHandler(handler)
140 logger.addHandler(handler)
141
141
142 def close_all(self):
142 def close_all(self):
143 """Close and remove all connection objects being kept for reuse."""
143 """Close and remove all connection objects being kept for reuse."""
144 for openconns in self._connections.values():
144 for openconns in self._connections.values():
145 for conn in openconns:
145 for conn in openconns:
146 conn.close()
146 conn.close()
147 self._connections = {}
147 self._connections = {}
148
148
149 # shamelessly borrowed from urllib2.AbstractHTTPHandler
149 # shamelessly borrowed from urllib2.AbstractHTTPHandler
150 def do_open(self, http_class, req, use_ssl):
150 def do_open(self, http_class, req, use_ssl):
151 """Return an addinfourl object for the request, using http_class.
151 """Return an addinfourl object for the request, using http_class.
152
152
153 http_class must implement the HTTPConnection API from httplib.
153 http_class must implement the HTTPConnection API from httplib.
154 The addinfourl return value is a file-like object. It also
154 The addinfourl return value is a file-like object. It also
155 has methods and attributes including:
155 has methods and attributes including:
156 - info(): return a mimetools.Message object for the headers
156 - info(): return a mimetools.Message object for the headers
157 - geturl(): return the original request URL
157 - geturl(): return the original request URL
158 - code: HTTP status code
158 - code: HTTP status code
159 """
159 """
160 # If using a proxy, the host returned by get_host() is
160 # If using a proxy, the host returned by get_host() is
161 # actually the proxy. On Python 2.6.1, the real destination
161 # actually the proxy. On Python 2.6.1, the real destination
162 # hostname is encoded in the URI in the urllib2 request
162 # hostname is encoded in the URI in the urllib2 request
163 # object. On Python 2.6.5, it's stored in the _tunnel_host
163 # object. On Python 2.6.5, it's stored in the _tunnel_host
164 # attribute which has no accessor.
164 # attribute which has no accessor.
165 tunhost = getattr(req, '_tunnel_host', None)
165 tunhost = getattr(req, '_tunnel_host', None)
166 host = req.get_host()
166 host = req.get_host()
167 if tunhost:
167 if tunhost:
168 proxyhost = host
168 proxyhost = host
169 host = tunhost
169 host = tunhost
170 elif req.has_proxy():
170 elif req.has_proxy():
171 proxyhost = req.get_host()
171 proxyhost = req.get_host()
172 host = req.get_selector().split('://', 1)[1].split('/', 1)[0]
172 host = req.get_selector().split('://', 1)[1].split('/', 1)[0]
173 else:
173 else:
174 proxyhost = None
174 proxyhost = None
175
175
176 if proxyhost:
176 if proxyhost:
177 if ':' in proxyhost:
177 if ':' in proxyhost:
178 # Note: this means we'll explode if we try and use an
178 # Note: this means we'll explode if we try and use an
179 # IPv6 http proxy. This isn't a regression, so we
179 # IPv6 http proxy. This isn't a regression, so we
180 # won't worry about it for now.
180 # won't worry about it for now.
181 proxyhost, proxyport = proxyhost.rsplit(':', 1)
181 proxyhost, proxyport = proxyhost.rsplit(':', 1)
182 else:
182 else:
183 proxyport = 3128 # squid default
183 proxyport = 3128 # squid default
184 proxy = (proxyhost, proxyport)
184 proxy = (proxyhost, proxyport)
185 else:
185 else:
186 proxy = None
186 proxy = None
187
187
188 if not host:
188 if not host:
189 raise urllib2.URLError('no host given')
189 raise urllib2.URLError('no host given')
190
190
191 connkey = use_ssl, host, proxy
191 connkey = use_ssl, host, proxy
192 allconns = self._connections.get(connkey, [])
192 allconns = self._connections.get(connkey, [])
193 conns = [c for c in allconns if not c.busy()]
193 conns = [c for c in allconns if not c.busy()]
194 if conns:
194 if conns:
195 h = conns[0]
195 h = conns[0]
196 else:
196 else:
197 if allconns:
197 if allconns:
198 self.ui.debug('all connections for %s busy, making a new '
198 self.ui.debug('all connections for %s busy, making a new '
199 'one\n' % host)
199 'one\n' % host)
200 timeout = None
200 timeout = None
201 if req.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
201 if req.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
202 timeout = req.timeout
202 timeout = req.timeout
203 h = http_class(host, timeout=timeout, proxy_hostport=proxy)
203 h = http_class(host, timeout=timeout, proxy_hostport=proxy)
204 self._connections.setdefault(connkey, []).append(h)
204 self._connections.setdefault(connkey, []).append(h)
205
205
206 headers = dict(req.headers)
206 headers = dict(req.headers)
207 headers.update(req.unredirected_hdrs)
207 headers.update(req.unredirected_hdrs)
208 headers = dict(
208 headers = dict(
209 (name.title(), val) for name, val in headers.items())
209 (name.title(), val) for name, val in headers.items())
210 try:
210 try:
211 path = req.get_selector()
211 path = req.get_selector()
212 if '://' in path:
212 if '://' in path:
213 path = path.split('://', 1)[1].split('/', 1)[1]
213 path = path.split('://', 1)[1].split('/', 1)[1]
214 if path[0] != '/':
214 if path[0] != '/':
215 path = '/' + path
215 path = '/' + path
216 h.request(req.get_method(), path, req.data, headers)
216 h.request(req.get_method(), path, req.data, headers)
217 r = h.getresponse()
217 r = h.getresponse()
218 except socket.error, err: # XXX what error?
218 except socket.error, err: # XXX what error?
219 raise urllib2.URLError(err)
219 raise urllib2.URLError(err)
220
220
221 # Pick apart the HTTPResponse object to get the addinfourl
221 # Pick apart the HTTPResponse object to get the addinfourl
222 # object initialized properly.
222 # object initialized properly.
223 r.recv = r.read
223 r.recv = r.read
224
224
225 resp = urllib.addinfourl(r, r.headers, req.get_full_url())
225 resp = urllib.addinfourl(r, r.headers, req.get_full_url())
226 resp.code = r.status
226 resp.code = r.status
227 resp.msg = r.reason
227 resp.msg = r.reason
228 return resp
228 return resp
229
229
230 # httplib always uses the given host/port as the socket connect
230 # httplib always uses the given host/port as the socket connect
231 # target, and then allows full URIs in the request path, which it
231 # target, and then allows full URIs in the request path, which it
232 # then observes and treats as a signal to do proxying instead.
232 # then observes and treats as a signal to do proxying instead.
233 def http_open(self, req):
233 def http_open(self, req):
234 if req.get_full_url().startswith('https'):
234 if req.get_full_url().startswith('https'):
235 return self.https_open(req)
235 return self.https_open(req)
236 def makehttpcon(*args, **kwargs):
236 def makehttpcon(*args, **kwargs):
237 return HTTPConnection(*args, use_ssl=False, **kwargs)
237 k2 = dict(kwargs)
238 k2['use_ssl'] = False
239 return HTTPConnection(*args, **k2)
238 return self.do_open(makehttpcon, req, False)
240 return self.do_open(makehttpcon, req, False)
239
241
240 def https_open(self, req):
242 def https_open(self, req):
241 # req.get_full_url() does not contain credentials and we may
243 # req.get_full_url() does not contain credentials and we may
242 # need them to match the certificates.
244 # need them to match the certificates.
243 url = req.get_full_url()
245 url = req.get_full_url()
244 user, password = self.pwmgr.find_stored_password(url)
246 user, password = self.pwmgr.find_stored_password(url)
245 res = readauthforuri(self.ui, url, user)
247 res = readauthforuri(self.ui, url, user)
246 if res:
248 if res:
247 group, auth = res
249 group, auth = res
248 self.auth = auth
250 self.auth = auth
249 self.ui.debug("using auth.%s.* for authentication\n" % group)
251 self.ui.debug("using auth.%s.* for authentication\n" % group)
250 else:
252 else:
251 self.auth = None
253 self.auth = None
252 return self.do_open(self._makesslconnection, req, True)
254 return self.do_open(self._makesslconnection, req, True)
253
255
254 def _makesslconnection(self, host, port=443, *args, **kwargs):
256 def _makesslconnection(self, host, port=443, *args, **kwargs):
255 keyfile = None
257 keyfile = None
256 certfile = None
258 certfile = None
257
259
258 if args: # key_file
260 if args: # key_file
259 keyfile = args.pop(0)
261 keyfile = args.pop(0)
260 if args: # cert_file
262 if args: # cert_file
261 certfile = args.pop(0)
263 certfile = args.pop(0)
262
264
263 # if the user has specified different key/cert files in
265 # if the user has specified different key/cert files in
264 # hgrc, we prefer these
266 # hgrc, we prefer these
265 if self.auth and 'key' in self.auth and 'cert' in self.auth:
267 if self.auth and 'key' in self.auth and 'cert' in self.auth:
266 keyfile = self.auth['key']
268 keyfile = self.auth['key']
267 certfile = self.auth['cert']
269 certfile = self.auth['cert']
268
270
269 # let host port take precedence
271 # let host port take precedence
270 if ':' in host and '[' not in host or ']:' in host:
272 if ':' in host and '[' not in host or ']:' in host:
271 host, port = host.rsplit(':', 1)
273 host, port = host.rsplit(':', 1)
272 port = int(port)
274 port = int(port)
273 if '[' in host:
275 if '[' in host:
274 host = host[1:-1]
276 host = host[1:-1]
275
277
276 if keyfile:
278 if keyfile:
277 kwargs['keyfile'] = keyfile
279 kwargs['keyfile'] = keyfile
278 if certfile:
280 if certfile:
279 kwargs['certfile'] = certfile
281 kwargs['certfile'] = certfile
280
282
281 kwargs.update(sslutil.sslkwargs(self.ui, host))
283 kwargs.update(sslutil.sslkwargs(self.ui, host))
282
284
283 con = HTTPConnection(host, port, use_ssl=True,
285 con = HTTPConnection(host, port, use_ssl=True,
284 ssl_validator=sslutil.validator(self.ui, host),
286 ssl_validator=sslutil.validator(self.ui, host),
285 **kwargs)
287 **kwargs)
286 return con
288 return con
General Comments 0
You need to be logged in to leave comments. Login now