##// END OF EJS Templates
HTTPS: fix python2.3, persistent connections, don't explode if SSL is not available...
Alexis S. L. Carvalho -
r2569:52ce0d6b default
parent child Browse files
Show More
@@ -1,325 +1,334 b''
1 # httprepo.py - HTTP repository proxy classes for mercurial
1 # httprepo.py - HTTP repository proxy classes for mercurial
2 #
2 #
3 # Copyright 2005 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005 Matt Mackall <mpm@selenic.com>
4 #
4 #
5 # This software may be used and distributed according to the terms
5 # This software may be used and distributed according to the terms
6 # of the GNU General Public License, incorporated herein by reference.
6 # of the GNU General Public License, incorporated herein by reference.
7
7
8 from node import *
8 from node import *
9 from remoterepo import *
9 from remoterepo import *
10 from i18n import gettext as _
10 from i18n import gettext as _
11 from demandload import *
11 from demandload import *
12 demandload(globals(), "hg os urllib urllib2 urlparse zlib util httplib")
12 demandload(globals(), "hg os urllib urllib2 urlparse zlib util httplib")
13 demandload(globals(), "errno keepalive tempfile socket")
13 demandload(globals(), "errno keepalive tempfile socket")
14
14
15 class passwordmgr(urllib2.HTTPPasswordMgrWithDefaultRealm):
15 class passwordmgr(urllib2.HTTPPasswordMgrWithDefaultRealm):
16 def __init__(self, ui):
16 def __init__(self, ui):
17 urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self)
17 urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self)
18 self.ui = ui
18 self.ui = ui
19
19
20 def find_user_password(self, realm, authuri):
20 def find_user_password(self, realm, authuri):
21 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password(
21 authinfo = urllib2.HTTPPasswordMgrWithDefaultRealm.find_user_password(
22 self, realm, authuri)
22 self, realm, authuri)
23 user, passwd = authinfo
23 user, passwd = authinfo
24 if user and passwd:
24 if user and passwd:
25 return (user, passwd)
25 return (user, passwd)
26
26
27 if not self.ui.interactive:
27 if not self.ui.interactive:
28 raise util.Abort(_('http authorization required'))
28 raise util.Abort(_('http authorization required'))
29
29
30 self.ui.write(_("http authorization required\n"))
30 self.ui.write(_("http authorization required\n"))
31 self.ui.status(_("realm: %s\n") % realm)
31 self.ui.status(_("realm: %s\n") % realm)
32 if user:
32 if user:
33 self.ui.status(_("user: %s\n") % user)
33 self.ui.status(_("user: %s\n") % user)
34 else:
34 else:
35 user = self.ui.prompt(_("user:"), default=None)
35 user = self.ui.prompt(_("user:"), default=None)
36
36
37 if not passwd:
37 if not passwd:
38 passwd = self.ui.getpass()
38 passwd = self.ui.getpass()
39
39
40 self.add_password(realm, authuri, user, passwd)
40 self.add_password(realm, authuri, user, passwd)
41 return (user, passwd)
41 return (user, passwd)
42
42
43 def netlocsplit(netloc):
43 def netlocsplit(netloc):
44 '''split [user[:passwd]@]host[:port] into 4-tuple.'''
44 '''split [user[:passwd]@]host[:port] into 4-tuple.'''
45
45
46 a = netloc.find('@')
46 a = netloc.find('@')
47 if a == -1:
47 if a == -1:
48 user, passwd = None, None
48 user, passwd = None, None
49 else:
49 else:
50 userpass, netloc = netloc[:a], netloc[a+1:]
50 userpass, netloc = netloc[:a], netloc[a+1:]
51 c = userpass.find(':')
51 c = userpass.find(':')
52 if c == -1:
52 if c == -1:
53 user, passwd = urllib.unquote(userpass), None
53 user, passwd = urllib.unquote(userpass), None
54 else:
54 else:
55 user = urllib.unquote(userpass[:c])
55 user = urllib.unquote(userpass[:c])
56 passwd = urllib.unquote(userpass[c+1:])
56 passwd = urllib.unquote(userpass[c+1:])
57 c = netloc.find(':')
57 c = netloc.find(':')
58 if c == -1:
58 if c == -1:
59 host, port = netloc, None
59 host, port = netloc, None
60 else:
60 else:
61 host, port = netloc[:c], netloc[c+1:]
61 host, port = netloc[:c], netloc[c+1:]
62 return host, port, user, passwd
62 return host, port, user, passwd
63
63
64 def netlocunsplit(host, port, user=None, passwd=None):
64 def netlocunsplit(host, port, user=None, passwd=None):
65 '''turn host, port, user, passwd into [user[:passwd]@]host[:port].'''
65 '''turn host, port, user, passwd into [user[:passwd]@]host[:port].'''
66 if port:
66 if port:
67 hostport = host + ':' + port
67 hostport = host + ':' + port
68 else:
68 else:
69 hostport = host
69 hostport = host
70 if user:
70 if user:
71 if passwd:
71 if passwd:
72 userpass = urllib.quote(user) + ':' + urllib.quote(passwd)
72 userpass = urllib.quote(user) + ':' + urllib.quote(passwd)
73 else:
73 else:
74 userpass = urllib.quote(user)
74 userpass = urllib.quote(user)
75 return userpass + '@' + hostport
75 return userpass + '@' + hostport
76 return hostport
76 return hostport
77
77
78 class httpconnection(keepalive.HTTPConnection):
78 class httpconnection(keepalive.HTTPConnection):
79 # must be able to send big bundle as stream.
79 # must be able to send big bundle as stream.
80
80
81 def send(self, data):
81 def send(self, data):
82 if isinstance(data, str):
82 if isinstance(data, str):
83 keepalive.HTTPConnection.send(self, data)
83 keepalive.HTTPConnection.send(self, data)
84 else:
84 else:
85 # if auth required, some data sent twice, so rewind here
85 # if auth required, some data sent twice, so rewind here
86 data.seek(0)
86 data.seek(0)
87 for chunk in util.filechunkiter(data):
87 for chunk in util.filechunkiter(data):
88 keepalive.HTTPConnection.send(self, chunk)
88 keepalive.HTTPConnection.send(self, chunk)
89
89
90 class httphandler(keepalive.HTTPHandler):
90 class basehttphandler(keepalive.HTTPHandler):
91 def http_open(self, req):
91 def http_open(self, req):
92 return self.do_open(httpconnection, req)
92 return self.do_open(httpconnection, req)
93
93
94 class httpsconnection(httplib.HTTPSConnection):
94 has_https = hasattr(urllib2, 'HTTPSHandler')
95 # must be able to send big bundle as stream.
95 if has_https:
96 class httpsconnection(httplib.HTTPSConnection):
97 response_class = keepalive.HTTPResponse
98 # must be able to send big bundle as stream.
96
99
97 def send(self, data):
100 def send(self, data):
98 if isinstance(data, str):
101 if isinstance(data, str):
99 httplib.HTTPSConnection.send(self, data)
102 httplib.HTTPSConnection.send(self, data)
100 else:
103 else:
101 # if auth required, some data sent twice, so rewind here
104 # if auth required, some data sent twice, so rewind here
102 data.seek(0)
105 data.seek(0)
103 for chunk in util.filechunkiter(data):
106 for chunk in util.filechunkiter(data):
104 httplib.HTTPSConnection.send(self, chunk)
107 httplib.HTTPSConnection.send(self, chunk)
105
108
106 class httpshandler(urllib2.HTTPSHandler):
109 class httphandler(basehttphandler, urllib2.HTTPSHandler):
107 def https_open(self, req):
110 def https_open(self, req):
108 return self.do_open(httpsconnection, req)
111 return self.do_open(httpsconnection, req)
112 else:
113 class httphandler(basehttphandler):
114 pass
109
115
110 class httprepository(remoterepository):
116 class httprepository(remoterepository):
111 def __init__(self, ui, path):
117 def __init__(self, ui, path):
112 self.caps = None
118 self.caps = None
113 scheme, netloc, urlpath, query, frag = urlparse.urlsplit(path)
119 scheme, netloc, urlpath, query, frag = urlparse.urlsplit(path)
114 if query or frag:
120 if query or frag:
115 raise util.Abort(_('unsupported URL component: "%s"') %
121 raise util.Abort(_('unsupported URL component: "%s"') %
116 (query or frag))
122 (query or frag))
117 if not urlpath: urlpath = '/'
123 if not urlpath: urlpath = '/'
118 host, port, user, passwd = netlocsplit(netloc)
124 host, port, user, passwd = netlocsplit(netloc)
119
125
120 # urllib cannot handle URLs with embedded user or passwd
126 # urllib cannot handle URLs with embedded user or passwd
121 self.url = urlparse.urlunsplit((scheme, netlocunsplit(host, port),
127 self.url = urlparse.urlunsplit((scheme, netlocunsplit(host, port),
122 urlpath, '', ''))
128 urlpath, '', ''))
123 self.ui = ui
129 self.ui = ui
124
130
125 proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy')
131 proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy')
126 proxyauthinfo = None
132 proxyauthinfo = None
127 handler = httphandler()
133 handler = httphandler()
128
134
129 if proxyurl:
135 if proxyurl:
130 # proxy can be proper url or host[:port]
136 # proxy can be proper url or host[:port]
131 if not (proxyurl.startswith('http:') or
137 if not (proxyurl.startswith('http:') or
132 proxyurl.startswith('https:')):
138 proxyurl.startswith('https:')):
133 proxyurl = 'http://' + proxyurl + '/'
139 proxyurl = 'http://' + proxyurl + '/'
134 snpqf = urlparse.urlsplit(proxyurl)
140 snpqf = urlparse.urlsplit(proxyurl)
135 proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf
141 proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf
136 hpup = netlocsplit(proxynetloc)
142 hpup = netlocsplit(proxynetloc)
137
143
138 proxyhost, proxyport, proxyuser, proxypasswd = hpup
144 proxyhost, proxyport, proxyuser, proxypasswd = hpup
139 if not proxyuser:
145 if not proxyuser:
140 proxyuser = ui.config("http_proxy", "user")
146 proxyuser = ui.config("http_proxy", "user")
141 proxypasswd = ui.config("http_proxy", "passwd")
147 proxypasswd = ui.config("http_proxy", "passwd")
142
148
143 # see if we should use a proxy for this url
149 # see if we should use a proxy for this url
144 no_list = [ "localhost", "127.0.0.1" ]
150 no_list = [ "localhost", "127.0.0.1" ]
145 no_list.extend([p.lower() for
151 no_list.extend([p.lower() for
146 p in ui.configlist("http_proxy", "no")])
152 p in ui.configlist("http_proxy", "no")])
147 no_list.extend([p.strip().lower() for
153 no_list.extend([p.strip().lower() for
148 p in os.getenv("no_proxy", '').split(',')
154 p in os.getenv("no_proxy", '').split(',')
149 if p.strip()])
155 if p.strip()])
150 # "http_proxy.always" config is for running tests on localhost
156 # "http_proxy.always" config is for running tests on localhost
151 if (not ui.configbool("http_proxy", "always") and
157 if (not ui.configbool("http_proxy", "always") and
152 host.lower() in no_list):
158 host.lower() in no_list):
153 ui.debug(_('disabling proxy for %s\n') % host)
159 ui.debug(_('disabling proxy for %s\n') % host)
154 else:
160 else:
155 proxyurl = urlparse.urlunsplit((
161 proxyurl = urlparse.urlunsplit((
156 proxyscheme, netlocunsplit(proxyhost, proxyport,
162 proxyscheme, netlocunsplit(proxyhost, proxyport,
157 proxyuser, proxypasswd or ''),
163 proxyuser, proxypasswd or ''),
158 proxypath, proxyquery, proxyfrag))
164 proxypath, proxyquery, proxyfrag))
159 handler = urllib2.ProxyHandler({scheme: proxyurl})
165 handler = urllib2.ProxyHandler({scheme: proxyurl})
160 ui.debug(_('proxying through %s\n') % proxyurl)
166 ui.debug(_('proxying through %s\n') % proxyurl)
161
167
162 # urllib2 takes proxy values from the environment and those
168 # urllib2 takes proxy values from the environment and those
163 # will take precedence if found, so drop them
169 # will take precedence if found, so drop them
164 for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]:
170 for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]:
165 try:
171 try:
166 if os.environ.has_key(env):
172 if os.environ.has_key(env):
167 del os.environ[env]
173 del os.environ[env]
168 except OSError:
174 except OSError:
169 pass
175 pass
170
176
171 passmgr = passwordmgr(ui)
177 passmgr = passwordmgr(ui)
172 if user:
178 if user:
173 ui.debug(_('http auth: user %s, password %s\n') %
179 ui.debug(_('http auth: user %s, password %s\n') %
174 (user, passwd and '*' * len(passwd) or 'not set'))
180 (user, passwd and '*' * len(passwd) or 'not set'))
175 passmgr.add_password(None, host, user, passwd or '')
181 passmgr.add_password(None, host, user, passwd or '')
176
182
177 opener = urllib2.build_opener(
183 opener = urllib2.build_opener(
178 handler,
184 handler,
179 httpshandler(),
180 urllib2.HTTPBasicAuthHandler(passmgr),
185 urllib2.HTTPBasicAuthHandler(passmgr),
181 urllib2.HTTPDigestAuthHandler(passmgr))
186 urllib2.HTTPDigestAuthHandler(passmgr))
182
187
183 # 1.0 here is the _protocol_ version
188 # 1.0 here is the _protocol_ version
184 opener.addheaders = [('User-agent', 'mercurial/proto-1.0')]
189 opener.addheaders = [('User-agent', 'mercurial/proto-1.0')]
185 urllib2.install_opener(opener)
190 urllib2.install_opener(opener)
186
191
187 # look up capabilities only when needed
192 # look up capabilities only when needed
188
193
189 def get_caps(self):
194 def get_caps(self):
190 if self.caps is None:
195 if self.caps is None:
191 try:
196 try:
192 self.caps = self.do_read('capabilities').split()
197 self.caps = self.do_read('capabilities').split()
193 except hg.RepoError:
198 except hg.RepoError:
194 self.caps = ()
199 self.caps = ()
195 self.ui.debug(_('capabilities: %s\n') %
200 self.ui.debug(_('capabilities: %s\n') %
196 (' '.join(self.caps or ['none'])))
201 (' '.join(self.caps or ['none'])))
197 return self.caps
202 return self.caps
198
203
199 capabilities = property(get_caps)
204 capabilities = property(get_caps)
200
205
201 def lock(self):
206 def lock(self):
202 raise util.Abort(_('operation not supported over http'))
207 raise util.Abort(_('operation not supported over http'))
203
208
204 def do_cmd(self, cmd, **args):
209 def do_cmd(self, cmd, **args):
205 data = args.pop('data', None)
210 data = args.pop('data', None)
206 headers = args.pop('headers', {})
211 headers = args.pop('headers', {})
207 self.ui.debug(_("sending %s command\n") % cmd)
212 self.ui.debug(_("sending %s command\n") % cmd)
208 q = {"cmd": cmd}
213 q = {"cmd": cmd}
209 q.update(args)
214 q.update(args)
210 qs = urllib.urlencode(q)
215 qs = urllib.urlencode(q)
211 cu = "%s?%s" % (self.url, qs)
216 cu = "%s?%s" % (self.url, qs)
212 try:
217 try:
213 resp = urllib2.urlopen(urllib2.Request(cu, data, headers))
218 resp = urllib2.urlopen(urllib2.Request(cu, data, headers))
214 except urllib2.HTTPError, inst:
219 except urllib2.HTTPError, inst:
215 if inst.code == 401:
220 if inst.code == 401:
216 raise util.Abort(_('authorization failed'))
221 raise util.Abort(_('authorization failed'))
217 raise
222 raise
218 except httplib.HTTPException, inst:
223 except httplib.HTTPException, inst:
219 self.ui.debug(_('http error while sending %s command\n') % cmd)
224 self.ui.debug(_('http error while sending %s command\n') % cmd)
220 self.ui.print_exc()
225 self.ui.print_exc()
221 raise IOError(None, inst)
226 raise IOError(None, inst)
222 try:
227 try:
223 proto = resp.getheader('content-type')
228 proto = resp.getheader('content-type')
224 except AttributeError:
229 except AttributeError:
225 proto = resp.headers['content-type']
230 proto = resp.headers['content-type']
226
231
227 # accept old "text/plain" and "application/hg-changegroup" for now
232 # accept old "text/plain" and "application/hg-changegroup" for now
228 if not proto.startswith('application/mercurial') and \
233 if not proto.startswith('application/mercurial') and \
229 not proto.startswith('text/plain') and \
234 not proto.startswith('text/plain') and \
230 not proto.startswith('application/hg-changegroup'):
235 not proto.startswith('application/hg-changegroup'):
231 raise hg.RepoError(_("'%s' does not appear to be an hg repository") %
236 raise hg.RepoError(_("'%s' does not appear to be an hg repository") %
232 self.url)
237 self.url)
233
238
234 if proto.startswith('application/mercurial'):
239 if proto.startswith('application/mercurial'):
235 version = proto[22:]
240 version = proto[22:]
236 if float(version) > 0.1:
241 if float(version) > 0.1:
237 raise hg.RepoError(_("'%s' uses newer protocol %s") %
242 raise hg.RepoError(_("'%s' uses newer protocol %s") %
238 (self.url, version))
243 (self.url, version))
239
244
240 return resp
245 return resp
241
246
242 def do_read(self, cmd, **args):
247 def do_read(self, cmd, **args):
243 fp = self.do_cmd(cmd, **args)
248 fp = self.do_cmd(cmd, **args)
244 try:
249 try:
245 return fp.read()
250 return fp.read()
246 finally:
251 finally:
247 # if using keepalive, allow connection to be reused
252 # if using keepalive, allow connection to be reused
248 fp.close()
253 fp.close()
249
254
250 def heads(self):
255 def heads(self):
251 d = self.do_read("heads")
256 d = self.do_read("heads")
252 try:
257 try:
253 return map(bin, d[:-1].split(" "))
258 return map(bin, d[:-1].split(" "))
254 except:
259 except:
255 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
260 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
256 raise
261 raise
257
262
258 def branches(self, nodes):
263 def branches(self, nodes):
259 n = " ".join(map(hex, nodes))
264 n = " ".join(map(hex, nodes))
260 d = self.do_read("branches", nodes=n)
265 d = self.do_read("branches", nodes=n)
261 try:
266 try:
262 br = [ tuple(map(bin, b.split(" "))) for b in d.splitlines() ]
267 br = [ tuple(map(bin, b.split(" "))) for b in d.splitlines() ]
263 return br
268 return br
264 except:
269 except:
265 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
270 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
266 raise
271 raise
267
272
268 def between(self, pairs):
273 def between(self, pairs):
269 n = "\n".join(["-".join(map(hex, p)) for p in pairs])
274 n = "\n".join(["-".join(map(hex, p)) for p in pairs])
270 d = self.do_read("between", pairs=n)
275 d = self.do_read("between", pairs=n)
271 try:
276 try:
272 p = [ l and map(bin, l.split(" ")) or [] for l in d.splitlines() ]
277 p = [ l and map(bin, l.split(" ")) or [] for l in d.splitlines() ]
273 return p
278 return p
274 except:
279 except:
275 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
280 self.ui.warn(_("unexpected response:\n") + d[:400] + "\n...\n")
276 raise
281 raise
277
282
278 def changegroup(self, nodes, kind):
283 def changegroup(self, nodes, kind):
279 n = " ".join(map(hex, nodes))
284 n = " ".join(map(hex, nodes))
280 f = self.do_cmd("changegroup", roots=n)
285 f = self.do_cmd("changegroup", roots=n)
281 bytes = 0
286 bytes = 0
282
287
283 def zgenerator(f):
288 def zgenerator(f):
284 zd = zlib.decompressobj()
289 zd = zlib.decompressobj()
285 try:
290 try:
286 for chnk in f:
291 for chnk in f:
287 yield zd.decompress(chnk)
292 yield zd.decompress(chnk)
288 except httplib.HTTPException, inst:
293 except httplib.HTTPException, inst:
289 raise IOError(None, _('connection ended unexpectedly'))
294 raise IOError(None, _('connection ended unexpectedly'))
290 yield zd.flush()
295 yield zd.flush()
291
296
292 return util.chunkbuffer(zgenerator(util.filechunkiter(f)))
297 return util.chunkbuffer(zgenerator(util.filechunkiter(f)))
293
298
294 def unbundle(self, cg, heads, source):
299 def unbundle(self, cg, heads, source):
295 # have to stream bundle to a temp file because we do not have
300 # have to stream bundle to a temp file because we do not have
296 # http 1.1 chunked transfer.
301 # http 1.1 chunked transfer.
297
302
298 fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
303 fd, tempname = tempfile.mkstemp(prefix='hg-unbundle-')
299 fp = os.fdopen(fd, 'wb+')
304 fp = os.fdopen(fd, 'wb+')
300 try:
305 try:
301 for chunk in util.filechunkiter(cg):
306 for chunk in util.filechunkiter(cg):
302 fp.write(chunk)
307 fp.write(chunk)
303 length = fp.tell()
308 length = fp.tell()
304 try:
309 try:
305 rfp = self.do_cmd(
310 rfp = self.do_cmd(
306 'unbundle', data=fp,
311 'unbundle', data=fp,
307 headers={'content-length': length,
312 headers={'content-length': length,
308 'content-type': 'application/octet-stream'},
313 'content-type': 'application/octet-stream'},
309 heads=' '.join(map(hex, heads)))
314 heads=' '.join(map(hex, heads)))
310 try:
315 try:
311 ret = int(rfp.readline())
316 ret = int(rfp.readline())
312 self.ui.write(rfp.read())
317 self.ui.write(rfp.read())
313 return ret
318 return ret
314 finally:
319 finally:
315 rfp.close()
320 rfp.close()
316 except socket.error, err:
321 except socket.error, err:
317 if err[0] in (errno.ECONNRESET, errno.EPIPE):
322 if err[0] in (errno.ECONNRESET, errno.EPIPE):
318 raise util.Abort(_('push failed: %s'), err[1])
323 raise util.Abort(_('push failed: %s'), err[1])
319 raise util.Abort(err[1])
324 raise util.Abort(err[1])
320 finally:
325 finally:
321 fp.close()
326 fp.close()
322 os.unlink(tempname)
327 os.unlink(tempname)
323
328
324 class httpsrepository(httprepository):
329 class httpsrepository(httprepository):
325 pass
330 def __init__(self, ui, path):
331 if not has_https:
332 raise util.Abort(_('Python support for SSL and HTTPS '
333 'is not installed'))
334 httprepository.__init__(self, ui, path)
General Comments 0
You need to be logged in to leave comments. Login now