##// END OF EJS Templates
wireprotov2: re-add tuple around `bundle2` check...
Raphaël Gomès -
r46668:aad81032 default
parent child Browse files
Show More
@@ -1,1121 +1,1121 b''
1 # httppeer.py - HTTP repository proxy classes for mercurial
1 # httppeer.py - HTTP repository proxy classes for mercurial
2 #
2 #
3 # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
3 # Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
4 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
5 #
5 #
6 # This software may be used and distributed according to the terms of the
6 # This software may be used and distributed according to the terms of the
7 # GNU General Public License version 2 or any later version.
7 # GNU General Public License version 2 or any later version.
8
8
9 from __future__ import absolute_import
9 from __future__ import absolute_import
10
10
11 import errno
11 import errno
12 import io
12 import io
13 import os
13 import os
14 import socket
14 import socket
15 import struct
15 import struct
16 import weakref
16 import weakref
17
17
18 from .i18n import _
18 from .i18n import _
19 from .pycompat import getattr
19 from .pycompat import getattr
20 from . import (
20 from . import (
21 bundle2,
21 bundle2,
22 error,
22 error,
23 httpconnection,
23 httpconnection,
24 pycompat,
24 pycompat,
25 statichttprepo,
25 statichttprepo,
26 url as urlmod,
26 url as urlmod,
27 util,
27 util,
28 wireprotoframing,
28 wireprotoframing,
29 wireprototypes,
29 wireprototypes,
30 wireprotov1peer,
30 wireprotov1peer,
31 wireprotov2peer,
31 wireprotov2peer,
32 wireprotov2server,
32 wireprotov2server,
33 )
33 )
34 from .interfaces import (
34 from .interfaces import (
35 repository,
35 repository,
36 util as interfaceutil,
36 util as interfaceutil,
37 )
37 )
38 from .utils import (
38 from .utils import (
39 cborutil,
39 cborutil,
40 stringutil,
40 stringutil,
41 )
41 )
42
42
43 httplib = util.httplib
43 httplib = util.httplib
44 urlerr = util.urlerr
44 urlerr = util.urlerr
45 urlreq = util.urlreq
45 urlreq = util.urlreq
46
46
47
47
48 def encodevalueinheaders(value, header, limit):
48 def encodevalueinheaders(value, header, limit):
49 """Encode a string value into multiple HTTP headers.
49 """Encode a string value into multiple HTTP headers.
50
50
51 ``value`` will be encoded into 1 or more HTTP headers with the names
51 ``value`` will be encoded into 1 or more HTTP headers with the names
52 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
52 ``header-<N>`` where ``<N>`` is an integer starting at 1. Each header
53 name + value will be at most ``limit`` bytes long.
53 name + value will be at most ``limit`` bytes long.
54
54
55 Returns an iterable of 2-tuples consisting of header names and
55 Returns an iterable of 2-tuples consisting of header names and
56 values as native strings.
56 values as native strings.
57 """
57 """
58 # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
58 # HTTP Headers are ASCII. Python 3 requires them to be unicodes,
59 # not bytes. This function always takes bytes in as arguments.
59 # not bytes. This function always takes bytes in as arguments.
60 fmt = pycompat.strurl(header) + r'-%s'
60 fmt = pycompat.strurl(header) + r'-%s'
61 # Note: it is *NOT* a bug that the last bit here is a bytestring
61 # Note: it is *NOT* a bug that the last bit here is a bytestring
62 # and not a unicode: we're just getting the encoded length anyway,
62 # and not a unicode: we're just getting the encoded length anyway,
63 # and using an r-string to make it portable between Python 2 and 3
63 # and using an r-string to make it portable between Python 2 and 3
64 # doesn't work because then the \r is a literal backslash-r
64 # doesn't work because then the \r is a literal backslash-r
65 # instead of a carriage return.
65 # instead of a carriage return.
66 valuelen = limit - len(fmt % '000') - len(b': \r\n')
66 valuelen = limit - len(fmt % '000') - len(b': \r\n')
67 result = []
67 result = []
68
68
69 n = 0
69 n = 0
70 for i in pycompat.xrange(0, len(value), valuelen):
70 for i in pycompat.xrange(0, len(value), valuelen):
71 n += 1
71 n += 1
72 result.append((fmt % str(n), pycompat.strurl(value[i : i + valuelen])))
72 result.append((fmt % str(n), pycompat.strurl(value[i : i + valuelen])))
73
73
74 return result
74 return result
75
75
76
76
77 class _multifile(object):
77 class _multifile(object):
78 def __init__(self, *fileobjs):
78 def __init__(self, *fileobjs):
79 for f in fileobjs:
79 for f in fileobjs:
80 if not util.safehasattr(f, b'length'):
80 if not util.safehasattr(f, b'length'):
81 raise ValueError(
81 raise ValueError(
82 b'_multifile only supports file objects that '
82 b'_multifile only supports file objects that '
83 b'have a length but this one does not:',
83 b'have a length but this one does not:',
84 type(f),
84 type(f),
85 f,
85 f,
86 )
86 )
87 self._fileobjs = fileobjs
87 self._fileobjs = fileobjs
88 self._index = 0
88 self._index = 0
89
89
90 @property
90 @property
91 def length(self):
91 def length(self):
92 return sum(f.length for f in self._fileobjs)
92 return sum(f.length for f in self._fileobjs)
93
93
94 def read(self, amt=None):
94 def read(self, amt=None):
95 if amt <= 0:
95 if amt <= 0:
96 return b''.join(f.read() for f in self._fileobjs)
96 return b''.join(f.read() for f in self._fileobjs)
97 parts = []
97 parts = []
98 while amt and self._index < len(self._fileobjs):
98 while amt and self._index < len(self._fileobjs):
99 parts.append(self._fileobjs[self._index].read(amt))
99 parts.append(self._fileobjs[self._index].read(amt))
100 got = len(parts[-1])
100 got = len(parts[-1])
101 if got < amt:
101 if got < amt:
102 self._index += 1
102 self._index += 1
103 amt -= got
103 amt -= got
104 return b''.join(parts)
104 return b''.join(parts)
105
105
106 def seek(self, offset, whence=os.SEEK_SET):
106 def seek(self, offset, whence=os.SEEK_SET):
107 if whence != os.SEEK_SET:
107 if whence != os.SEEK_SET:
108 raise NotImplementedError(
108 raise NotImplementedError(
109 b'_multifile does not support anything other'
109 b'_multifile does not support anything other'
110 b' than os.SEEK_SET for whence on seek()'
110 b' than os.SEEK_SET for whence on seek()'
111 )
111 )
112 if offset != 0:
112 if offset != 0:
113 raise NotImplementedError(
113 raise NotImplementedError(
114 b'_multifile only supports seeking to start, but that '
114 b'_multifile only supports seeking to start, but that '
115 b'could be fixed if you need it'
115 b'could be fixed if you need it'
116 )
116 )
117 for f in self._fileobjs:
117 for f in self._fileobjs:
118 f.seek(0)
118 f.seek(0)
119 self._index = 0
119 self._index = 0
120
120
121
121
122 def makev1commandrequest(
122 def makev1commandrequest(
123 ui, requestbuilder, caps, capablefn, repobaseurl, cmd, args
123 ui, requestbuilder, caps, capablefn, repobaseurl, cmd, args
124 ):
124 ):
125 """Make an HTTP request to run a command for a version 1 client.
125 """Make an HTTP request to run a command for a version 1 client.
126
126
127 ``caps`` is a set of known server capabilities. The value may be
127 ``caps`` is a set of known server capabilities. The value may be
128 None if capabilities are not yet known.
128 None if capabilities are not yet known.
129
129
130 ``capablefn`` is a function to evaluate a capability.
130 ``capablefn`` is a function to evaluate a capability.
131
131
132 ``cmd``, ``args``, and ``data`` define the command, its arguments, and
132 ``cmd``, ``args``, and ``data`` define the command, its arguments, and
133 raw data to pass to it.
133 raw data to pass to it.
134 """
134 """
135 if cmd == b'pushkey':
135 if cmd == b'pushkey':
136 args[b'data'] = b''
136 args[b'data'] = b''
137 data = args.pop(b'data', None)
137 data = args.pop(b'data', None)
138 headers = args.pop(b'headers', {})
138 headers = args.pop(b'headers', {})
139
139
140 ui.debug(b"sending %s command\n" % cmd)
140 ui.debug(b"sending %s command\n" % cmd)
141 q = [(b'cmd', cmd)]
141 q = [(b'cmd', cmd)]
142 headersize = 0
142 headersize = 0
143 # Important: don't use self.capable() here or else you end up
143 # Important: don't use self.capable() here or else you end up
144 # with infinite recursion when trying to look up capabilities
144 # with infinite recursion when trying to look up capabilities
145 # for the first time.
145 # for the first time.
146 postargsok = caps is not None and b'httppostargs' in caps
146 postargsok = caps is not None and b'httppostargs' in caps
147
147
148 # Send arguments via POST.
148 # Send arguments via POST.
149 if postargsok and args:
149 if postargsok and args:
150 strargs = urlreq.urlencode(sorted(args.items()))
150 strargs = urlreq.urlencode(sorted(args.items()))
151 if not data:
151 if not data:
152 data = strargs
152 data = strargs
153 else:
153 else:
154 if isinstance(data, bytes):
154 if isinstance(data, bytes):
155 i = io.BytesIO(data)
155 i = io.BytesIO(data)
156 i.length = len(data)
156 i.length = len(data)
157 data = i
157 data = i
158 argsio = io.BytesIO(strargs)
158 argsio = io.BytesIO(strargs)
159 argsio.length = len(strargs)
159 argsio.length = len(strargs)
160 data = _multifile(argsio, data)
160 data = _multifile(argsio, data)
161 headers['X-HgArgs-Post'] = len(strargs)
161 headers['X-HgArgs-Post'] = len(strargs)
162 elif args:
162 elif args:
163 # Calling self.capable() can infinite loop if we are calling
163 # Calling self.capable() can infinite loop if we are calling
164 # "capabilities". But that command should never accept wire
164 # "capabilities". But that command should never accept wire
165 # protocol arguments. So this should never happen.
165 # protocol arguments. So this should never happen.
166 assert cmd != b'capabilities'
166 assert cmd != b'capabilities'
167 httpheader = capablefn(b'httpheader')
167 httpheader = capablefn(b'httpheader')
168 if httpheader:
168 if httpheader:
169 headersize = int(httpheader.split(b',', 1)[0])
169 headersize = int(httpheader.split(b',', 1)[0])
170
170
171 # Send arguments via HTTP headers.
171 # Send arguments via HTTP headers.
172 if headersize > 0:
172 if headersize > 0:
173 # The headers can typically carry more data than the URL.
173 # The headers can typically carry more data than the URL.
174 encargs = urlreq.urlencode(sorted(args.items()))
174 encargs = urlreq.urlencode(sorted(args.items()))
175 for header, value in encodevalueinheaders(
175 for header, value in encodevalueinheaders(
176 encargs, b'X-HgArg', headersize
176 encargs, b'X-HgArg', headersize
177 ):
177 ):
178 headers[header] = value
178 headers[header] = value
179 # Send arguments via query string (Mercurial <1.9).
179 # Send arguments via query string (Mercurial <1.9).
180 else:
180 else:
181 q += sorted(args.items())
181 q += sorted(args.items())
182
182
183 qs = b'?%s' % urlreq.urlencode(q)
183 qs = b'?%s' % urlreq.urlencode(q)
184 cu = b"%s%s" % (repobaseurl, qs)
184 cu = b"%s%s" % (repobaseurl, qs)
185 size = 0
185 size = 0
186 if util.safehasattr(data, b'length'):
186 if util.safehasattr(data, b'length'):
187 size = data.length
187 size = data.length
188 elif data is not None:
188 elif data is not None:
189 size = len(data)
189 size = len(data)
190 if data is not None and 'Content-Type' not in headers:
190 if data is not None and 'Content-Type' not in headers:
191 headers['Content-Type'] = 'application/mercurial-0.1'
191 headers['Content-Type'] = 'application/mercurial-0.1'
192
192
193 # Tell the server we accept application/mercurial-0.2 and multiple
193 # Tell the server we accept application/mercurial-0.2 and multiple
194 # compression formats if the server is capable of emitting those
194 # compression formats if the server is capable of emitting those
195 # payloads.
195 # payloads.
196 # Note: Keep this set empty by default, as client advertisement of
196 # Note: Keep this set empty by default, as client advertisement of
197 # protocol parameters should only occur after the handshake.
197 # protocol parameters should only occur after the handshake.
198 protoparams = set()
198 protoparams = set()
199
199
200 mediatypes = set()
200 mediatypes = set()
201 if caps is not None:
201 if caps is not None:
202 mt = capablefn(b'httpmediatype')
202 mt = capablefn(b'httpmediatype')
203 if mt:
203 if mt:
204 protoparams.add(b'0.1')
204 protoparams.add(b'0.1')
205 mediatypes = set(mt.split(b','))
205 mediatypes = set(mt.split(b','))
206
206
207 protoparams.add(b'partial-pull')
207 protoparams.add(b'partial-pull')
208
208
209 if b'0.2tx' in mediatypes:
209 if b'0.2tx' in mediatypes:
210 protoparams.add(b'0.2')
210 protoparams.add(b'0.2')
211
211
212 if b'0.2tx' in mediatypes and capablefn(b'compression'):
212 if b'0.2tx' in mediatypes and capablefn(b'compression'):
213 # We /could/ compare supported compression formats and prune
213 # We /could/ compare supported compression formats and prune
214 # non-mutually supported or error if nothing is mutually supported.
214 # non-mutually supported or error if nothing is mutually supported.
215 # For now, send the full list to the server and have it error.
215 # For now, send the full list to the server and have it error.
216 comps = [
216 comps = [
217 e.wireprotosupport().name
217 e.wireprotosupport().name
218 for e in util.compengines.supportedwireengines(util.CLIENTROLE)
218 for e in util.compengines.supportedwireengines(util.CLIENTROLE)
219 ]
219 ]
220 protoparams.add(b'comp=%s' % b','.join(comps))
220 protoparams.add(b'comp=%s' % b','.join(comps))
221
221
222 if protoparams:
222 if protoparams:
223 protoheaders = encodevalueinheaders(
223 protoheaders = encodevalueinheaders(
224 b' '.join(sorted(protoparams)), b'X-HgProto', headersize or 1024
224 b' '.join(sorted(protoparams)), b'X-HgProto', headersize or 1024
225 )
225 )
226 for header, value in protoheaders:
226 for header, value in protoheaders:
227 headers[header] = value
227 headers[header] = value
228
228
229 varyheaders = []
229 varyheaders = []
230 for header in headers:
230 for header in headers:
231 if header.lower().startswith('x-hg'):
231 if header.lower().startswith('x-hg'):
232 varyheaders.append(header)
232 varyheaders.append(header)
233
233
234 if varyheaders:
234 if varyheaders:
235 headers['Vary'] = ','.join(sorted(varyheaders))
235 headers['Vary'] = ','.join(sorted(varyheaders))
236
236
237 req = requestbuilder(pycompat.strurl(cu), data, headers)
237 req = requestbuilder(pycompat.strurl(cu), data, headers)
238
238
239 if data is not None:
239 if data is not None:
240 ui.debug(b"sending %d bytes\n" % size)
240 ui.debug(b"sending %d bytes\n" % size)
241 req.add_unredirected_header('Content-Length', '%d' % size)
241 req.add_unredirected_header('Content-Length', '%d' % size)
242
242
243 return req, cu, qs
243 return req, cu, qs
244
244
245
245
246 def _reqdata(req):
246 def _reqdata(req):
247 """Get request data, if any. If no data, returns None."""
247 """Get request data, if any. If no data, returns None."""
248 if pycompat.ispy3:
248 if pycompat.ispy3:
249 return req.data
249 return req.data
250 if not req.has_data():
250 if not req.has_data():
251 return None
251 return None
252 return req.get_data()
252 return req.get_data()
253
253
254
254
255 def sendrequest(ui, opener, req):
255 def sendrequest(ui, opener, req):
256 """Send a prepared HTTP request.
256 """Send a prepared HTTP request.
257
257
258 Returns the response object.
258 Returns the response object.
259 """
259 """
260 dbg = ui.debug
260 dbg = ui.debug
261 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
261 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
262 line = b'devel-peer-request: %s\n'
262 line = b'devel-peer-request: %s\n'
263 dbg(
263 dbg(
264 line
264 line
265 % b'%s %s'
265 % b'%s %s'
266 % (
266 % (
267 pycompat.bytesurl(req.get_method()),
267 pycompat.bytesurl(req.get_method()),
268 pycompat.bytesurl(req.get_full_url()),
268 pycompat.bytesurl(req.get_full_url()),
269 )
269 )
270 )
270 )
271 hgargssize = None
271 hgargssize = None
272
272
273 for header, value in sorted(req.header_items()):
273 for header, value in sorted(req.header_items()):
274 header = pycompat.bytesurl(header)
274 header = pycompat.bytesurl(header)
275 value = pycompat.bytesurl(value)
275 value = pycompat.bytesurl(value)
276 if header.startswith(b'X-hgarg-'):
276 if header.startswith(b'X-hgarg-'):
277 if hgargssize is None:
277 if hgargssize is None:
278 hgargssize = 0
278 hgargssize = 0
279 hgargssize += len(value)
279 hgargssize += len(value)
280 else:
280 else:
281 dbg(line % b' %s %s' % (header, value))
281 dbg(line % b' %s %s' % (header, value))
282
282
283 if hgargssize is not None:
283 if hgargssize is not None:
284 dbg(
284 dbg(
285 line
285 line
286 % b' %d bytes of commands arguments in headers'
286 % b' %d bytes of commands arguments in headers'
287 % hgargssize
287 % hgargssize
288 )
288 )
289 data = _reqdata(req)
289 data = _reqdata(req)
290 if data is not None:
290 if data is not None:
291 length = getattr(data, 'length', None)
291 length = getattr(data, 'length', None)
292 if length is None:
292 if length is None:
293 length = len(data)
293 length = len(data)
294 dbg(line % b' %d bytes of data' % length)
294 dbg(line % b' %d bytes of data' % length)
295
295
296 start = util.timer()
296 start = util.timer()
297
297
298 res = None
298 res = None
299 try:
299 try:
300 res = opener.open(req)
300 res = opener.open(req)
301 except urlerr.httperror as inst:
301 except urlerr.httperror as inst:
302 if inst.code == 401:
302 if inst.code == 401:
303 raise error.Abort(_(b'authorization failed'))
303 raise error.Abort(_(b'authorization failed'))
304 raise
304 raise
305 except httplib.HTTPException as inst:
305 except httplib.HTTPException as inst:
306 ui.debug(
306 ui.debug(
307 b'http error requesting %s\n'
307 b'http error requesting %s\n'
308 % util.hidepassword(req.get_full_url())
308 % util.hidepassword(req.get_full_url())
309 )
309 )
310 ui.traceback()
310 ui.traceback()
311 raise IOError(None, inst)
311 raise IOError(None, inst)
312 finally:
312 finally:
313 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
313 if ui.debugflag and ui.configbool(b'devel', b'debug.peer-request'):
314 code = res.code if res else -1
314 code = res.code if res else -1
315 dbg(
315 dbg(
316 line
316 line
317 % b' finished in %.4f seconds (%d)'
317 % b' finished in %.4f seconds (%d)'
318 % (util.timer() - start, code)
318 % (util.timer() - start, code)
319 )
319 )
320
320
321 # Insert error handlers for common I/O failures.
321 # Insert error handlers for common I/O failures.
322 urlmod.wrapresponse(res)
322 urlmod.wrapresponse(res)
323
323
324 return res
324 return res
325
325
326
326
327 class RedirectedRepoError(error.RepoError):
327 class RedirectedRepoError(error.RepoError):
328 def __init__(self, msg, respurl):
328 def __init__(self, msg, respurl):
329 super(RedirectedRepoError, self).__init__(msg)
329 super(RedirectedRepoError, self).__init__(msg)
330 self.respurl = respurl
330 self.respurl = respurl
331
331
332
332
333 def parsev1commandresponse(
333 def parsev1commandresponse(
334 ui, baseurl, requrl, qs, resp, compressible, allowcbor=False
334 ui, baseurl, requrl, qs, resp, compressible, allowcbor=False
335 ):
335 ):
336 # record the url we got redirected to
336 # record the url we got redirected to
337 redirected = False
337 redirected = False
338 respurl = pycompat.bytesurl(resp.geturl())
338 respurl = pycompat.bytesurl(resp.geturl())
339 if respurl.endswith(qs):
339 if respurl.endswith(qs):
340 respurl = respurl[: -len(qs)]
340 respurl = respurl[: -len(qs)]
341 qsdropped = False
341 qsdropped = False
342 else:
342 else:
343 qsdropped = True
343 qsdropped = True
344
344
345 if baseurl.rstrip(b'/') != respurl.rstrip(b'/'):
345 if baseurl.rstrip(b'/') != respurl.rstrip(b'/'):
346 redirected = True
346 redirected = True
347 if not ui.quiet:
347 if not ui.quiet:
348 ui.warn(_(b'real URL is %s\n') % respurl)
348 ui.warn(_(b'real URL is %s\n') % respurl)
349
349
350 try:
350 try:
351 proto = pycompat.bytesurl(resp.getheader('content-type', ''))
351 proto = pycompat.bytesurl(resp.getheader('content-type', ''))
352 except AttributeError:
352 except AttributeError:
353 proto = pycompat.bytesurl(resp.headers.get('content-type', ''))
353 proto = pycompat.bytesurl(resp.headers.get('content-type', ''))
354
354
355 safeurl = util.hidepassword(baseurl)
355 safeurl = util.hidepassword(baseurl)
356 if proto.startswith(b'application/hg-error'):
356 if proto.startswith(b'application/hg-error'):
357 raise error.OutOfBandError(resp.read())
357 raise error.OutOfBandError(resp.read())
358
358
359 # Pre 1.0 versions of Mercurial used text/plain and
359 # Pre 1.0 versions of Mercurial used text/plain and
360 # application/hg-changegroup. We don't support such old servers.
360 # application/hg-changegroup. We don't support such old servers.
361 if not proto.startswith(b'application/mercurial-'):
361 if not proto.startswith(b'application/mercurial-'):
362 ui.debug(b"requested URL: '%s'\n" % util.hidepassword(requrl))
362 ui.debug(b"requested URL: '%s'\n" % util.hidepassword(requrl))
363 msg = _(
363 msg = _(
364 b"'%s' does not appear to be an hg repository:\n"
364 b"'%s' does not appear to be an hg repository:\n"
365 b"---%%<--- (%s)\n%s\n---%%<---\n"
365 b"---%%<--- (%s)\n%s\n---%%<---\n"
366 ) % (safeurl, proto or b'no content-type', resp.read(1024))
366 ) % (safeurl, proto or b'no content-type', resp.read(1024))
367
367
368 # Some servers may strip the query string from the redirect. We
368 # Some servers may strip the query string from the redirect. We
369 # raise a special error type so callers can react to this specially.
369 # raise a special error type so callers can react to this specially.
370 if redirected and qsdropped:
370 if redirected and qsdropped:
371 raise RedirectedRepoError(msg, respurl)
371 raise RedirectedRepoError(msg, respurl)
372 else:
372 else:
373 raise error.RepoError(msg)
373 raise error.RepoError(msg)
374
374
375 try:
375 try:
376 subtype = proto.split(b'-', 1)[1]
376 subtype = proto.split(b'-', 1)[1]
377
377
378 # Unless we end up supporting CBOR in the legacy wire protocol,
378 # Unless we end up supporting CBOR in the legacy wire protocol,
379 # this should ONLY be encountered for the initial capabilities
379 # this should ONLY be encountered for the initial capabilities
380 # request during handshake.
380 # request during handshake.
381 if subtype == b'cbor':
381 if subtype == b'cbor':
382 if allowcbor:
382 if allowcbor:
383 return respurl, proto, resp
383 return respurl, proto, resp
384 else:
384 else:
385 raise error.RepoError(
385 raise error.RepoError(
386 _(b'unexpected CBOR response from server')
386 _(b'unexpected CBOR response from server')
387 )
387 )
388
388
389 version_info = tuple([int(n) for n in subtype.split(b'.')])
389 version_info = tuple([int(n) for n in subtype.split(b'.')])
390 except ValueError:
390 except ValueError:
391 raise error.RepoError(
391 raise error.RepoError(
392 _(b"'%s' sent a broken Content-Type header (%s)") % (safeurl, proto)
392 _(b"'%s' sent a broken Content-Type header (%s)") % (safeurl, proto)
393 )
393 )
394
394
395 # TODO consider switching to a decompression reader that uses
395 # TODO consider switching to a decompression reader that uses
396 # generators.
396 # generators.
397 if version_info == (0, 1):
397 if version_info == (0, 1):
398 if compressible:
398 if compressible:
399 resp = util.compengines[b'zlib'].decompressorreader(resp)
399 resp = util.compengines[b'zlib'].decompressorreader(resp)
400
400
401 elif version_info == (0, 2):
401 elif version_info == (0, 2):
402 # application/mercurial-0.2 always identifies the compression
402 # application/mercurial-0.2 always identifies the compression
403 # engine in the payload header.
403 # engine in the payload header.
404 elen = struct.unpack(b'B', util.readexactly(resp, 1))[0]
404 elen = struct.unpack(b'B', util.readexactly(resp, 1))[0]
405 ename = util.readexactly(resp, elen)
405 ename = util.readexactly(resp, elen)
406 engine = util.compengines.forwiretype(ename)
406 engine = util.compengines.forwiretype(ename)
407
407
408 resp = engine.decompressorreader(resp)
408 resp = engine.decompressorreader(resp)
409 else:
409 else:
410 raise error.RepoError(
410 raise error.RepoError(
411 _(b"'%s' uses newer protocol %s") % (safeurl, subtype)
411 _(b"'%s' uses newer protocol %s") % (safeurl, subtype)
412 )
412 )
413
413
414 return respurl, proto, resp
414 return respurl, proto, resp
415
415
416
416
417 class httppeer(wireprotov1peer.wirepeer):
417 class httppeer(wireprotov1peer.wirepeer):
418 def __init__(self, ui, path, url, opener, requestbuilder, caps):
418 def __init__(self, ui, path, url, opener, requestbuilder, caps):
419 self.ui = ui
419 self.ui = ui
420 self._path = path
420 self._path = path
421 self._url = url
421 self._url = url
422 self._caps = caps
422 self._caps = caps
423 self.limitedarguments = caps is not None and b'httppostargs' not in caps
423 self.limitedarguments = caps is not None and b'httppostargs' not in caps
424 self._urlopener = opener
424 self._urlopener = opener
425 self._requestbuilder = requestbuilder
425 self._requestbuilder = requestbuilder
426
426
427 def __del__(self):
427 def __del__(self):
428 for h in self._urlopener.handlers:
428 for h in self._urlopener.handlers:
429 h.close()
429 h.close()
430 getattr(h, "close_all", lambda: None)()
430 getattr(h, "close_all", lambda: None)()
431
431
432 # Begin of ipeerconnection interface.
432 # Begin of ipeerconnection interface.
433
433
434 def url(self):
434 def url(self):
435 return self._path
435 return self._path
436
436
437 def local(self):
437 def local(self):
438 return None
438 return None
439
439
440 def peer(self):
440 def peer(self):
441 return self
441 return self
442
442
443 def canpush(self):
443 def canpush(self):
444 return True
444 return True
445
445
446 def close(self):
446 def close(self):
447 try:
447 try:
448 reqs, sent, recv = (
448 reqs, sent, recv = (
449 self._urlopener.requestscount,
449 self._urlopener.requestscount,
450 self._urlopener.sentbytescount,
450 self._urlopener.sentbytescount,
451 self._urlopener.receivedbytescount,
451 self._urlopener.receivedbytescount,
452 )
452 )
453 except AttributeError:
453 except AttributeError:
454 return
454 return
455 self.ui.note(
455 self.ui.note(
456 _(
456 _(
457 b'(sent %d HTTP requests and %d bytes; '
457 b'(sent %d HTTP requests and %d bytes; '
458 b'received %d bytes in responses)\n'
458 b'received %d bytes in responses)\n'
459 )
459 )
460 % (reqs, sent, recv)
460 % (reqs, sent, recv)
461 )
461 )
462
462
463 # End of ipeerconnection interface.
463 # End of ipeerconnection interface.
464
464
465 # Begin of ipeercommands interface.
465 # Begin of ipeercommands interface.
466
466
467 def capabilities(self):
467 def capabilities(self):
468 return self._caps
468 return self._caps
469
469
470 # End of ipeercommands interface.
470 # End of ipeercommands interface.
471
471
472 def _callstream(self, cmd, _compressible=False, **args):
472 def _callstream(self, cmd, _compressible=False, **args):
473 args = pycompat.byteskwargs(args)
473 args = pycompat.byteskwargs(args)
474
474
475 req, cu, qs = makev1commandrequest(
475 req, cu, qs = makev1commandrequest(
476 self.ui,
476 self.ui,
477 self._requestbuilder,
477 self._requestbuilder,
478 self._caps,
478 self._caps,
479 self.capable,
479 self.capable,
480 self._url,
480 self._url,
481 cmd,
481 cmd,
482 args,
482 args,
483 )
483 )
484
484
485 resp = sendrequest(self.ui, self._urlopener, req)
485 resp = sendrequest(self.ui, self._urlopener, req)
486
486
487 self._url, ct, resp = parsev1commandresponse(
487 self._url, ct, resp = parsev1commandresponse(
488 self.ui, self._url, cu, qs, resp, _compressible
488 self.ui, self._url, cu, qs, resp, _compressible
489 )
489 )
490
490
491 return resp
491 return resp
492
492
493 def _call(self, cmd, **args):
493 def _call(self, cmd, **args):
494 fp = self._callstream(cmd, **args)
494 fp = self._callstream(cmd, **args)
495 try:
495 try:
496 return fp.read()
496 return fp.read()
497 finally:
497 finally:
498 # if using keepalive, allow connection to be reused
498 # if using keepalive, allow connection to be reused
499 fp.close()
499 fp.close()
500
500
501 def _callpush(self, cmd, cg, **args):
501 def _callpush(self, cmd, cg, **args):
502 # have to stream bundle to a temp file because we do not have
502 # have to stream bundle to a temp file because we do not have
503 # http 1.1 chunked transfer.
503 # http 1.1 chunked transfer.
504
504
505 types = self.capable(b'unbundle')
505 types = self.capable(b'unbundle')
506 try:
506 try:
507 types = types.split(b',')
507 types = types.split(b',')
508 except AttributeError:
508 except AttributeError:
509 # servers older than d1b16a746db6 will send 'unbundle' as a
509 # servers older than d1b16a746db6 will send 'unbundle' as a
510 # boolean capability. They only support headerless/uncompressed
510 # boolean capability. They only support headerless/uncompressed
511 # bundles.
511 # bundles.
512 types = [b""]
512 types = [b""]
513 for x in types:
513 for x in types:
514 if x in bundle2.bundletypes:
514 if x in bundle2.bundletypes:
515 type = x
515 type = x
516 break
516 break
517
517
518 tempname = bundle2.writebundle(self.ui, cg, None, type)
518 tempname = bundle2.writebundle(self.ui, cg, None, type)
519 fp = httpconnection.httpsendfile(self.ui, tempname, b"rb")
519 fp = httpconnection.httpsendfile(self.ui, tempname, b"rb")
520 headers = {'Content-Type': 'application/mercurial-0.1'}
520 headers = {'Content-Type': 'application/mercurial-0.1'}
521
521
522 try:
522 try:
523 r = self._call(cmd, data=fp, headers=headers, **args)
523 r = self._call(cmd, data=fp, headers=headers, **args)
524 vals = r.split(b'\n', 1)
524 vals = r.split(b'\n', 1)
525 if len(vals) < 2:
525 if len(vals) < 2:
526 raise error.ResponseError(_(b"unexpected response:"), r)
526 raise error.ResponseError(_(b"unexpected response:"), r)
527 return vals
527 return vals
528 except urlerr.httperror:
528 except urlerr.httperror:
529 # Catch and re-raise these so we don't try and treat them
529 # Catch and re-raise these so we don't try and treat them
530 # like generic socket errors. They lack any values in
530 # like generic socket errors. They lack any values in
531 # .args on Python 3 which breaks our socket.error block.
531 # .args on Python 3 which breaks our socket.error block.
532 raise
532 raise
533 except socket.error as err:
533 except socket.error as err:
534 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
534 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
535 raise error.Abort(_(b'push failed: %s') % err.args[1])
535 raise error.Abort(_(b'push failed: %s') % err.args[1])
536 raise error.Abort(err.args[1])
536 raise error.Abort(err.args[1])
537 finally:
537 finally:
538 fp.close()
538 fp.close()
539 os.unlink(tempname)
539 os.unlink(tempname)
540
540
541 def _calltwowaystream(self, cmd, fp, **args):
541 def _calltwowaystream(self, cmd, fp, **args):
542 filename = None
542 filename = None
543 try:
543 try:
544 # dump bundle to disk
544 # dump bundle to disk
545 fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg")
545 fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg")
546 with os.fdopen(fd, "wb") as fh:
546 with os.fdopen(fd, "wb") as fh:
547 d = fp.read(4096)
547 d = fp.read(4096)
548 while d:
548 while d:
549 fh.write(d)
549 fh.write(d)
550 d = fp.read(4096)
550 d = fp.read(4096)
551 # start http push
551 # start http push
552 with httpconnection.httpsendfile(self.ui, filename, b"rb") as fp_:
552 with httpconnection.httpsendfile(self.ui, filename, b"rb") as fp_:
553 headers = {'Content-Type': 'application/mercurial-0.1'}
553 headers = {'Content-Type': 'application/mercurial-0.1'}
554 return self._callstream(cmd, data=fp_, headers=headers, **args)
554 return self._callstream(cmd, data=fp_, headers=headers, **args)
555 finally:
555 finally:
556 if filename is not None:
556 if filename is not None:
557 os.unlink(filename)
557 os.unlink(filename)
558
558
559 def _callcompressable(self, cmd, **args):
559 def _callcompressable(self, cmd, **args):
560 return self._callstream(cmd, _compressible=True, **args)
560 return self._callstream(cmd, _compressible=True, **args)
561
561
562 def _abort(self, exception):
562 def _abort(self, exception):
563 raise exception
563 raise exception
564
564
565
565
566 def sendv2request(
566 def sendv2request(
567 ui, opener, requestbuilder, apiurl, permission, requests, redirect
567 ui, opener, requestbuilder, apiurl, permission, requests, redirect
568 ):
568 ):
569 wireprotoframing.populatestreamencoders()
569 wireprotoframing.populatestreamencoders()
570
570
571 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
571 uiencoders = ui.configlist(b'experimental', b'httppeer.v2-encoder-order')
572
572
573 if uiencoders:
573 if uiencoders:
574 encoders = []
574 encoders = []
575
575
576 for encoder in uiencoders:
576 for encoder in uiencoders:
577 if encoder not in wireprotoframing.STREAM_ENCODERS:
577 if encoder not in wireprotoframing.STREAM_ENCODERS:
578 ui.warn(
578 ui.warn(
579 _(
579 _(
580 b'wire protocol version 2 encoder referenced in '
580 b'wire protocol version 2 encoder referenced in '
581 b'config (%s) is not known; ignoring\n'
581 b'config (%s) is not known; ignoring\n'
582 )
582 )
583 % encoder
583 % encoder
584 )
584 )
585 else:
585 else:
586 encoders.append(encoder)
586 encoders.append(encoder)
587
587
588 else:
588 else:
589 encoders = wireprotoframing.STREAM_ENCODERS_ORDER
589 encoders = wireprotoframing.STREAM_ENCODERS_ORDER
590
590
591 reactor = wireprotoframing.clientreactor(
591 reactor = wireprotoframing.clientreactor(
592 ui,
592 ui,
593 hasmultiplesend=False,
593 hasmultiplesend=False,
594 buffersends=True,
594 buffersends=True,
595 clientcontentencoders=encoders,
595 clientcontentencoders=encoders,
596 )
596 )
597
597
598 handler = wireprotov2peer.clienthandler(
598 handler = wireprotov2peer.clienthandler(
599 ui, reactor, opener=opener, requestbuilder=requestbuilder
599 ui, reactor, opener=opener, requestbuilder=requestbuilder
600 )
600 )
601
601
602 url = b'%s/%s' % (apiurl, permission)
602 url = b'%s/%s' % (apiurl, permission)
603
603
604 if len(requests) > 1:
604 if len(requests) > 1:
605 url += b'/multirequest'
605 url += b'/multirequest'
606 else:
606 else:
607 url += b'/%s' % requests[0][0]
607 url += b'/%s' % requests[0][0]
608
608
609 ui.debug(b'sending %d commands\n' % len(requests))
609 ui.debug(b'sending %d commands\n' % len(requests))
610 for command, args, f in requests:
610 for command, args, f in requests:
611 ui.debug(
611 ui.debug(
612 b'sending command %s: %s\n'
612 b'sending command %s: %s\n'
613 % (command, stringutil.pprint(args, indent=2))
613 % (command, stringutil.pprint(args, indent=2))
614 )
614 )
615 assert not list(
615 assert not list(
616 handler.callcommand(command, args, f, redirect=redirect)
616 handler.callcommand(command, args, f, redirect=redirect)
617 )
617 )
618
618
619 # TODO stream this.
619 # TODO stream this.
620 body = b''.join(map(bytes, handler.flushcommands()))
620 body = b''.join(map(bytes, handler.flushcommands()))
621
621
622 # TODO modify user-agent to reflect v2
622 # TODO modify user-agent to reflect v2
623 headers = {
623 headers = {
624 'Accept': wireprotov2server.FRAMINGTYPE,
624 'Accept': wireprotov2server.FRAMINGTYPE,
625 'Content-Type': wireprotov2server.FRAMINGTYPE,
625 'Content-Type': wireprotov2server.FRAMINGTYPE,
626 }
626 }
627
627
628 req = requestbuilder(pycompat.strurl(url), body, headers)
628 req = requestbuilder(pycompat.strurl(url), body, headers)
629 req.add_unredirected_header('Content-Length', '%d' % len(body))
629 req.add_unredirected_header('Content-Length', '%d' % len(body))
630
630
631 try:
631 try:
632 res = opener.open(req)
632 res = opener.open(req)
633 except urlerr.httperror as e:
633 except urlerr.httperror as e:
634 if e.code == 401:
634 if e.code == 401:
635 raise error.Abort(_(b'authorization failed'))
635 raise error.Abort(_(b'authorization failed'))
636
636
637 raise
637 raise
638 except httplib.HTTPException as e:
638 except httplib.HTTPException as e:
639 ui.traceback()
639 ui.traceback()
640 raise IOError(None, e)
640 raise IOError(None, e)
641
641
642 return handler, res
642 return handler, res
643
643
644
644
645 class queuedcommandfuture(pycompat.futures.Future):
645 class queuedcommandfuture(pycompat.futures.Future):
646 """Wraps result() on command futures to trigger submission on call."""
646 """Wraps result() on command futures to trigger submission on call."""
647
647
648 def result(self, timeout=None):
648 def result(self, timeout=None):
649 if self.done():
649 if self.done():
650 return pycompat.futures.Future.result(self, timeout)
650 return pycompat.futures.Future.result(self, timeout)
651
651
652 self._peerexecutor.sendcommands()
652 self._peerexecutor.sendcommands()
653
653
654 # sendcommands() will restore the original __class__ and self.result
654 # sendcommands() will restore the original __class__ and self.result
655 # will resolve to Future.result.
655 # will resolve to Future.result.
656 return self.result(timeout)
656 return self.result(timeout)
657
657
658
658
659 @interfaceutil.implementer(repository.ipeercommandexecutor)
659 @interfaceutil.implementer(repository.ipeercommandexecutor)
660 class httpv2executor(object):
660 class httpv2executor(object):
661 def __init__(
661 def __init__(
662 self, ui, opener, requestbuilder, apiurl, descriptor, redirect
662 self, ui, opener, requestbuilder, apiurl, descriptor, redirect
663 ):
663 ):
664 self._ui = ui
664 self._ui = ui
665 self._opener = opener
665 self._opener = opener
666 self._requestbuilder = requestbuilder
666 self._requestbuilder = requestbuilder
667 self._apiurl = apiurl
667 self._apiurl = apiurl
668 self._descriptor = descriptor
668 self._descriptor = descriptor
669 self._redirect = redirect
669 self._redirect = redirect
670 self._sent = False
670 self._sent = False
671 self._closed = False
671 self._closed = False
672 self._neededpermissions = set()
672 self._neededpermissions = set()
673 self._calls = []
673 self._calls = []
674 self._futures = weakref.WeakSet()
674 self._futures = weakref.WeakSet()
675 self._responseexecutor = None
675 self._responseexecutor = None
676 self._responsef = None
676 self._responsef = None
677
677
678 def __enter__(self):
678 def __enter__(self):
679 return self
679 return self
680
680
681 def __exit__(self, exctype, excvalue, exctb):
681 def __exit__(self, exctype, excvalue, exctb):
682 self.close()
682 self.close()
683
683
684 def callcommand(self, command, args):
684 def callcommand(self, command, args):
685 if self._sent:
685 if self._sent:
686 raise error.ProgrammingError(
686 raise error.ProgrammingError(
687 b'callcommand() cannot be used after commands are sent'
687 b'callcommand() cannot be used after commands are sent'
688 )
688 )
689
689
690 if self._closed:
690 if self._closed:
691 raise error.ProgrammingError(
691 raise error.ProgrammingError(
692 b'callcommand() cannot be used after close()'
692 b'callcommand() cannot be used after close()'
693 )
693 )
694
694
695 # The service advertises which commands are available. So if we attempt
695 # The service advertises which commands are available. So if we attempt
696 # to call an unknown command or pass an unknown argument, we can screen
696 # to call an unknown command or pass an unknown argument, we can screen
697 # for this.
697 # for this.
698 if command not in self._descriptor[b'commands']:
698 if command not in self._descriptor[b'commands']:
699 raise error.ProgrammingError(
699 raise error.ProgrammingError(
700 b'wire protocol command %s is not available' % command
700 b'wire protocol command %s is not available' % command
701 )
701 )
702
702
703 cmdinfo = self._descriptor[b'commands'][command]
703 cmdinfo = self._descriptor[b'commands'][command]
704 unknownargs = set(args.keys()) - set(cmdinfo.get(b'args', {}))
704 unknownargs = set(args.keys()) - set(cmdinfo.get(b'args', {}))
705
705
706 if unknownargs:
706 if unknownargs:
707 raise error.ProgrammingError(
707 raise error.ProgrammingError(
708 b'wire protocol command %s does not accept argument: %s'
708 b'wire protocol command %s does not accept argument: %s'
709 % (command, b', '.join(sorted(unknownargs)))
709 % (command, b', '.join(sorted(unknownargs)))
710 )
710 )
711
711
712 self._neededpermissions |= set(cmdinfo[b'permissions'])
712 self._neededpermissions |= set(cmdinfo[b'permissions'])
713
713
714 # TODO we /could/ also validate types here, since the API descriptor
714 # TODO we /could/ also validate types here, since the API descriptor
715 # includes types...
715 # includes types...
716
716
717 f = pycompat.futures.Future()
717 f = pycompat.futures.Future()
718
718
719 # Monkeypatch it so result() triggers sendcommands(), otherwise result()
719 # Monkeypatch it so result() triggers sendcommands(), otherwise result()
720 # could deadlock.
720 # could deadlock.
721 f.__class__ = queuedcommandfuture
721 f.__class__ = queuedcommandfuture
722 f._peerexecutor = self
722 f._peerexecutor = self
723
723
724 self._futures.add(f)
724 self._futures.add(f)
725 self._calls.append((command, args, f))
725 self._calls.append((command, args, f))
726
726
727 return f
727 return f
728
728
729 def sendcommands(self):
729 def sendcommands(self):
730 if self._sent:
730 if self._sent:
731 return
731 return
732
732
733 if not self._calls:
733 if not self._calls:
734 return
734 return
735
735
736 self._sent = True
736 self._sent = True
737
737
738 # Unhack any future types so caller sees a clean type and so we
738 # Unhack any future types so caller sees a clean type and so we
739 # break reference cycle.
739 # break reference cycle.
740 for f in self._futures:
740 for f in self._futures:
741 if isinstance(f, queuedcommandfuture):
741 if isinstance(f, queuedcommandfuture):
742 f.__class__ = pycompat.futures.Future
742 f.__class__ = pycompat.futures.Future
743 f._peerexecutor = None
743 f._peerexecutor = None
744
744
745 # Mark the future as running and filter out cancelled futures.
745 # Mark the future as running and filter out cancelled futures.
746 calls = [
746 calls = [
747 (command, args, f)
747 (command, args, f)
748 for command, args, f in self._calls
748 for command, args, f in self._calls
749 if f.set_running_or_notify_cancel()
749 if f.set_running_or_notify_cancel()
750 ]
750 ]
751
751
752 # Clear out references, prevent improper object usage.
752 # Clear out references, prevent improper object usage.
753 self._calls = None
753 self._calls = None
754
754
755 if not calls:
755 if not calls:
756 return
756 return
757
757
758 permissions = set(self._neededpermissions)
758 permissions = set(self._neededpermissions)
759
759
760 if b'push' in permissions and b'pull' in permissions:
760 if b'push' in permissions and b'pull' in permissions:
761 permissions.remove(b'pull')
761 permissions.remove(b'pull')
762
762
763 if len(permissions) > 1:
763 if len(permissions) > 1:
764 raise error.RepoError(
764 raise error.RepoError(
765 _(b'cannot make request requiring multiple permissions: %s')
765 _(b'cannot make request requiring multiple permissions: %s')
766 % _(b', ').join(sorted(permissions))
766 % _(b', ').join(sorted(permissions))
767 )
767 )
768
768
769 permission = {
769 permission = {
770 b'push': b'rw',
770 b'push': b'rw',
771 b'pull': b'ro',
771 b'pull': b'ro',
772 }[permissions.pop()]
772 }[permissions.pop()]
773
773
774 handler, resp = sendv2request(
774 handler, resp = sendv2request(
775 self._ui,
775 self._ui,
776 self._opener,
776 self._opener,
777 self._requestbuilder,
777 self._requestbuilder,
778 self._apiurl,
778 self._apiurl,
779 permission,
779 permission,
780 calls,
780 calls,
781 self._redirect,
781 self._redirect,
782 )
782 )
783
783
784 # TODO we probably want to validate the HTTP code, media type, etc.
784 # TODO we probably want to validate the HTTP code, media type, etc.
785
785
786 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
786 self._responseexecutor = pycompat.futures.ThreadPoolExecutor(1)
787 self._responsef = self._responseexecutor.submit(
787 self._responsef = self._responseexecutor.submit(
788 self._handleresponse, handler, resp
788 self._handleresponse, handler, resp
789 )
789 )
790
790
791 def close(self):
791 def close(self):
792 if self._closed:
792 if self._closed:
793 return
793 return
794
794
795 self.sendcommands()
795 self.sendcommands()
796
796
797 self._closed = True
797 self._closed = True
798
798
799 if not self._responsef:
799 if not self._responsef:
800 return
800 return
801
801
802 # TODO ^C here may not result in immediate program termination.
802 # TODO ^C here may not result in immediate program termination.
803
803
804 try:
804 try:
805 self._responsef.result()
805 self._responsef.result()
806 finally:
806 finally:
807 self._responseexecutor.shutdown(wait=True)
807 self._responseexecutor.shutdown(wait=True)
808 self._responsef = None
808 self._responsef = None
809 self._responseexecutor = None
809 self._responseexecutor = None
810
810
811 # If any of our futures are still in progress, mark them as
811 # If any of our futures are still in progress, mark them as
812 # errored, otherwise a result() could wait indefinitely.
812 # errored, otherwise a result() could wait indefinitely.
813 for f in self._futures:
813 for f in self._futures:
814 if not f.done():
814 if not f.done():
815 f.set_exception(
815 f.set_exception(
816 error.ResponseError(_(b'unfulfilled command response'))
816 error.ResponseError(_(b'unfulfilled command response'))
817 )
817 )
818
818
819 self._futures = None
819 self._futures = None
820
820
821 def _handleresponse(self, handler, resp):
821 def _handleresponse(self, handler, resp):
822 # Called in a thread to read the response.
822 # Called in a thread to read the response.
823
823
824 while handler.readdata(resp):
824 while handler.readdata(resp):
825 pass
825 pass
826
826
827
827
828 @interfaceutil.implementer(repository.ipeerv2)
828 @interfaceutil.implementer(repository.ipeerv2)
829 class httpv2peer(object):
829 class httpv2peer(object):
830
830
831 limitedarguments = False
831 limitedarguments = False
832
832
833 def __init__(
833 def __init__(
834 self, ui, repourl, apipath, opener, requestbuilder, apidescriptor
834 self, ui, repourl, apipath, opener, requestbuilder, apidescriptor
835 ):
835 ):
836 self.ui = ui
836 self.ui = ui
837 self.apidescriptor = apidescriptor
837 self.apidescriptor = apidescriptor
838
838
839 if repourl.endswith(b'/'):
839 if repourl.endswith(b'/'):
840 repourl = repourl[:-1]
840 repourl = repourl[:-1]
841
841
842 self._url = repourl
842 self._url = repourl
843 self._apipath = apipath
843 self._apipath = apipath
844 self._apiurl = b'%s/%s' % (repourl, apipath)
844 self._apiurl = b'%s/%s' % (repourl, apipath)
845 self._opener = opener
845 self._opener = opener
846 self._requestbuilder = requestbuilder
846 self._requestbuilder = requestbuilder
847
847
848 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
848 self._redirect = wireprotov2peer.supportedredirects(ui, apidescriptor)
849
849
850 # Start of ipeerconnection.
850 # Start of ipeerconnection.
851
851
852 def url(self):
852 def url(self):
853 return self._url
853 return self._url
854
854
855 def local(self):
855 def local(self):
856 return None
856 return None
857
857
858 def peer(self):
858 def peer(self):
859 return self
859 return self
860
860
861 def canpush(self):
861 def canpush(self):
862 # TODO change once implemented.
862 # TODO change once implemented.
863 return False
863 return False
864
864
865 def close(self):
865 def close(self):
866 self.ui.note(
866 self.ui.note(
867 _(
867 _(
868 b'(sent %d HTTP requests and %d bytes; '
868 b'(sent %d HTTP requests and %d bytes; '
869 b'received %d bytes in responses)\n'
869 b'received %d bytes in responses)\n'
870 )
870 )
871 % (
871 % (
872 self._opener.requestscount,
872 self._opener.requestscount,
873 self._opener.sentbytescount,
873 self._opener.sentbytescount,
874 self._opener.receivedbytescount,
874 self._opener.receivedbytescount,
875 )
875 )
876 )
876 )
877
877
878 # End of ipeerconnection.
878 # End of ipeerconnection.
879
879
880 # Start of ipeercapabilities.
880 # Start of ipeercapabilities.
881
881
882 def capable(self, name):
882 def capable(self, name):
883 # The capabilities used internally historically map to capabilities
883 # The capabilities used internally historically map to capabilities
884 # advertised from the "capabilities" wire protocol command. However,
884 # advertised from the "capabilities" wire protocol command. However,
885 # version 2 of that command works differently.
885 # version 2 of that command works differently.
886
886
887 # Maps to commands that are available.
887 # Maps to commands that are available.
888 if name in (
888 if name in (
889 b'branchmap',
889 b'branchmap',
890 b'getbundle',
890 b'getbundle',
891 b'known',
891 b'known',
892 b'lookup',
892 b'lookup',
893 b'pushkey',
893 b'pushkey',
894 ):
894 ):
895 return True
895 return True
896
896
897 # Other concepts.
897 # Other concepts.
898 if name in b'bundle2':
898 if name in (b'bundle2',):
899 return True
899 return True
900
900
901 # Alias command-* to presence of command of that name.
901 # Alias command-* to presence of command of that name.
902 if name.startswith(b'command-'):
902 if name.startswith(b'command-'):
903 return name[len(b'command-') :] in self.apidescriptor[b'commands']
903 return name[len(b'command-') :] in self.apidescriptor[b'commands']
904
904
905 return False
905 return False
906
906
907 def requirecap(self, name, purpose):
907 def requirecap(self, name, purpose):
908 if self.capable(name):
908 if self.capable(name):
909 return
909 return
910
910
911 raise error.CapabilityError(
911 raise error.CapabilityError(
912 _(
912 _(
913 b'cannot %s; client or remote repository does not support the '
913 b'cannot %s; client or remote repository does not support the '
914 b'\'%s\' capability'
914 b'\'%s\' capability'
915 )
915 )
916 % (purpose, name)
916 % (purpose, name)
917 )
917 )
918
918
919 # End of ipeercapabilities.
919 # End of ipeercapabilities.
920
920
921 def _call(self, name, **args):
921 def _call(self, name, **args):
922 with self.commandexecutor() as e:
922 with self.commandexecutor() as e:
923 return e.callcommand(name, args).result()
923 return e.callcommand(name, args).result()
924
924
925 def commandexecutor(self):
925 def commandexecutor(self):
926 return httpv2executor(
926 return httpv2executor(
927 self.ui,
927 self.ui,
928 self._opener,
928 self._opener,
929 self._requestbuilder,
929 self._requestbuilder,
930 self._apiurl,
930 self._apiurl,
931 self.apidescriptor,
931 self.apidescriptor,
932 self._redirect,
932 self._redirect,
933 )
933 )
934
934
935
935
936 # Registry of API service names to metadata about peers that handle it.
936 # Registry of API service names to metadata about peers that handle it.
937 #
937 #
938 # The following keys are meaningful:
938 # The following keys are meaningful:
939 #
939 #
940 # init
940 # init
941 # Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
941 # Callable receiving (ui, repourl, servicepath, opener, requestbuilder,
942 # apidescriptor) to create a peer.
942 # apidescriptor) to create a peer.
943 #
943 #
944 # priority
944 # priority
945 # Integer priority for the service. If we could choose from multiple
945 # Integer priority for the service. If we could choose from multiple
946 # services, we choose the one with the highest priority.
946 # services, we choose the one with the highest priority.
947 API_PEERS = {
947 API_PEERS = {
948 wireprototypes.HTTP_WIREPROTO_V2: {
948 wireprototypes.HTTP_WIREPROTO_V2: {
949 b'init': httpv2peer,
949 b'init': httpv2peer,
950 b'priority': 50,
950 b'priority': 50,
951 },
951 },
952 }
952 }
953
953
954
954
955 def performhandshake(ui, url, opener, requestbuilder):
955 def performhandshake(ui, url, opener, requestbuilder):
956 # The handshake is a request to the capabilities command.
956 # The handshake is a request to the capabilities command.
957
957
958 caps = None
958 caps = None
959
959
960 def capable(x):
960 def capable(x):
961 raise error.ProgrammingError(b'should not be called')
961 raise error.ProgrammingError(b'should not be called')
962
962
963 args = {}
963 args = {}
964
964
965 # The client advertises support for newer protocols by adding an
965 # The client advertises support for newer protocols by adding an
966 # X-HgUpgrade-* header with a list of supported APIs and an
966 # X-HgUpgrade-* header with a list of supported APIs and an
967 # X-HgProto-* header advertising which serializing formats it supports.
967 # X-HgProto-* header advertising which serializing formats it supports.
968 # We only support the HTTP version 2 transport and CBOR responses for
968 # We only support the HTTP version 2 transport and CBOR responses for
969 # now.
969 # now.
970 advertisev2 = ui.configbool(b'experimental', b'httppeer.advertise-v2')
970 advertisev2 = ui.configbool(b'experimental', b'httppeer.advertise-v2')
971
971
972 if advertisev2:
972 if advertisev2:
973 args[b'headers'] = {
973 args[b'headers'] = {
974 'X-HgProto-1': 'cbor',
974 'X-HgProto-1': 'cbor',
975 }
975 }
976
976
977 args[b'headers'].update(
977 args[b'headers'].update(
978 encodevalueinheaders(
978 encodevalueinheaders(
979 b' '.join(sorted(API_PEERS)),
979 b' '.join(sorted(API_PEERS)),
980 b'X-HgUpgrade',
980 b'X-HgUpgrade',
981 # We don't know the header limit this early.
981 # We don't know the header limit this early.
982 # So make it small.
982 # So make it small.
983 1024,
983 1024,
984 )
984 )
985 )
985 )
986
986
987 req, requrl, qs = makev1commandrequest(
987 req, requrl, qs = makev1commandrequest(
988 ui, requestbuilder, caps, capable, url, b'capabilities', args
988 ui, requestbuilder, caps, capable, url, b'capabilities', args
989 )
989 )
990 resp = sendrequest(ui, opener, req)
990 resp = sendrequest(ui, opener, req)
991
991
992 # The server may redirect us to the repo root, stripping the
992 # The server may redirect us to the repo root, stripping the
993 # ?cmd=capabilities query string from the URL. The server would likely
993 # ?cmd=capabilities query string from the URL. The server would likely
994 # return HTML in this case and ``parsev1commandresponse()`` would raise.
994 # return HTML in this case and ``parsev1commandresponse()`` would raise.
995 # We catch this special case and re-issue the capabilities request against
995 # We catch this special case and re-issue the capabilities request against
996 # the new URL.
996 # the new URL.
997 #
997 #
998 # We should ideally not do this, as a redirect that drops the query
998 # We should ideally not do this, as a redirect that drops the query
999 # string from the URL is arguably a server bug. (Garbage in, garbage out).
999 # string from the URL is arguably a server bug. (Garbage in, garbage out).
1000 # However, Mercurial clients for several years appeared to handle this
1000 # However, Mercurial clients for several years appeared to handle this
1001 # issue without behavior degradation. And according to issue 5860, it may
1001 # issue without behavior degradation. And according to issue 5860, it may
1002 # be a longstanding bug in some server implementations. So we allow a
1002 # be a longstanding bug in some server implementations. So we allow a
1003 # redirect that drops the query string to "just work."
1003 # redirect that drops the query string to "just work."
1004 try:
1004 try:
1005 respurl, ct, resp = parsev1commandresponse(
1005 respurl, ct, resp = parsev1commandresponse(
1006 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
1006 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
1007 )
1007 )
1008 except RedirectedRepoError as e:
1008 except RedirectedRepoError as e:
1009 req, requrl, qs = makev1commandrequest(
1009 req, requrl, qs = makev1commandrequest(
1010 ui, requestbuilder, caps, capable, e.respurl, b'capabilities', args
1010 ui, requestbuilder, caps, capable, e.respurl, b'capabilities', args
1011 )
1011 )
1012 resp = sendrequest(ui, opener, req)
1012 resp = sendrequest(ui, opener, req)
1013 respurl, ct, resp = parsev1commandresponse(
1013 respurl, ct, resp = parsev1commandresponse(
1014 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
1014 ui, url, requrl, qs, resp, compressible=False, allowcbor=advertisev2
1015 )
1015 )
1016
1016
1017 try:
1017 try:
1018 rawdata = resp.read()
1018 rawdata = resp.read()
1019 finally:
1019 finally:
1020 resp.close()
1020 resp.close()
1021
1021
1022 if not ct.startswith(b'application/mercurial-'):
1022 if not ct.startswith(b'application/mercurial-'):
1023 raise error.ProgrammingError(b'unexpected content-type: %s' % ct)
1023 raise error.ProgrammingError(b'unexpected content-type: %s' % ct)
1024
1024
1025 if advertisev2:
1025 if advertisev2:
1026 if ct == b'application/mercurial-cbor':
1026 if ct == b'application/mercurial-cbor':
1027 try:
1027 try:
1028 info = cborutil.decodeall(rawdata)[0]
1028 info = cborutil.decodeall(rawdata)[0]
1029 except cborutil.CBORDecodeError:
1029 except cborutil.CBORDecodeError:
1030 raise error.Abort(
1030 raise error.Abort(
1031 _(b'error decoding CBOR from remote server'),
1031 _(b'error decoding CBOR from remote server'),
1032 hint=_(
1032 hint=_(
1033 b'try again and consider contacting '
1033 b'try again and consider contacting '
1034 b'the server operator'
1034 b'the server operator'
1035 ),
1035 ),
1036 )
1036 )
1037
1037
1038 # We got a legacy response. That's fine.
1038 # We got a legacy response. That's fine.
1039 elif ct in (b'application/mercurial-0.1', b'application/mercurial-0.2'):
1039 elif ct in (b'application/mercurial-0.1', b'application/mercurial-0.2'):
1040 info = {b'v1capabilities': set(rawdata.split())}
1040 info = {b'v1capabilities': set(rawdata.split())}
1041
1041
1042 else:
1042 else:
1043 raise error.RepoError(
1043 raise error.RepoError(
1044 _(b'unexpected response type from server: %s') % ct
1044 _(b'unexpected response type from server: %s') % ct
1045 )
1045 )
1046 else:
1046 else:
1047 info = {b'v1capabilities': set(rawdata.split())}
1047 info = {b'v1capabilities': set(rawdata.split())}
1048
1048
1049 return respurl, info
1049 return respurl, info
1050
1050
1051
1051
1052 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
1052 def makepeer(ui, path, opener=None, requestbuilder=urlreq.request):
1053 """Construct an appropriate HTTP peer instance.
1053 """Construct an appropriate HTTP peer instance.
1054
1054
1055 ``opener`` is an ``url.opener`` that should be used to establish
1055 ``opener`` is an ``url.opener`` that should be used to establish
1056 connections, perform HTTP requests.
1056 connections, perform HTTP requests.
1057
1057
1058 ``requestbuilder`` is the type used for constructing HTTP requests.
1058 ``requestbuilder`` is the type used for constructing HTTP requests.
1059 It exists as an argument so extensions can override the default.
1059 It exists as an argument so extensions can override the default.
1060 """
1060 """
1061 u = util.url(path)
1061 u = util.url(path)
1062 if u.query or u.fragment:
1062 if u.query or u.fragment:
1063 raise error.Abort(
1063 raise error.Abort(
1064 _(b'unsupported URL component: "%s"') % (u.query or u.fragment)
1064 _(b'unsupported URL component: "%s"') % (u.query or u.fragment)
1065 )
1065 )
1066
1066
1067 # urllib cannot handle URLs with embedded user or passwd.
1067 # urllib cannot handle URLs with embedded user or passwd.
1068 url, authinfo = u.authinfo()
1068 url, authinfo = u.authinfo()
1069 ui.debug(b'using %s\n' % url)
1069 ui.debug(b'using %s\n' % url)
1070
1070
1071 opener = opener or urlmod.opener(ui, authinfo)
1071 opener = opener or urlmod.opener(ui, authinfo)
1072
1072
1073 respurl, info = performhandshake(ui, url, opener, requestbuilder)
1073 respurl, info = performhandshake(ui, url, opener, requestbuilder)
1074
1074
1075 # Given the intersection of APIs that both we and the server support,
1075 # Given the intersection of APIs that both we and the server support,
1076 # sort by their advertised priority and pick the first one.
1076 # sort by their advertised priority and pick the first one.
1077 #
1077 #
1078 # TODO consider making this request-based and interface driven. For
1078 # TODO consider making this request-based and interface driven. For
1079 # example, the caller could say "I want a peer that does X." It's quite
1079 # example, the caller could say "I want a peer that does X." It's quite
1080 # possible that not all peers would do that. Since we know the service
1080 # possible that not all peers would do that. Since we know the service
1081 # capabilities, we could filter out services not meeting the
1081 # capabilities, we could filter out services not meeting the
1082 # requirements. Possibly by consulting the interfaces defined by the
1082 # requirements. Possibly by consulting the interfaces defined by the
1083 # peer type.
1083 # peer type.
1084 apipeerchoices = set(info.get(b'apis', {}).keys()) & set(API_PEERS.keys())
1084 apipeerchoices = set(info.get(b'apis', {}).keys()) & set(API_PEERS.keys())
1085
1085
1086 preferredchoices = sorted(
1086 preferredchoices = sorted(
1087 apipeerchoices, key=lambda x: API_PEERS[x][b'priority'], reverse=True
1087 apipeerchoices, key=lambda x: API_PEERS[x][b'priority'], reverse=True
1088 )
1088 )
1089
1089
1090 for service in preferredchoices:
1090 for service in preferredchoices:
1091 apipath = b'%s/%s' % (info[b'apibase'].rstrip(b'/'), service)
1091 apipath = b'%s/%s' % (info[b'apibase'].rstrip(b'/'), service)
1092
1092
1093 return API_PEERS[service][b'init'](
1093 return API_PEERS[service][b'init'](
1094 ui, respurl, apipath, opener, requestbuilder, info[b'apis'][service]
1094 ui, respurl, apipath, opener, requestbuilder, info[b'apis'][service]
1095 )
1095 )
1096
1096
1097 # Failed to construct an API peer. Fall back to legacy.
1097 # Failed to construct an API peer. Fall back to legacy.
1098 return httppeer(
1098 return httppeer(
1099 ui, path, respurl, opener, requestbuilder, info[b'v1capabilities']
1099 ui, path, respurl, opener, requestbuilder, info[b'v1capabilities']
1100 )
1100 )
1101
1101
1102
1102
1103 def instance(ui, path, create, intents=None, createopts=None):
1103 def instance(ui, path, create, intents=None, createopts=None):
1104 if create:
1104 if create:
1105 raise error.Abort(_(b'cannot create new http repository'))
1105 raise error.Abort(_(b'cannot create new http repository'))
1106 try:
1106 try:
1107 if path.startswith(b'https:') and not urlmod.has_https:
1107 if path.startswith(b'https:') and not urlmod.has_https:
1108 raise error.Abort(
1108 raise error.Abort(
1109 _(b'Python support for SSL and HTTPS is not installed')
1109 _(b'Python support for SSL and HTTPS is not installed')
1110 )
1110 )
1111
1111
1112 inst = makepeer(ui, path)
1112 inst = makepeer(ui, path)
1113
1113
1114 return inst
1114 return inst
1115 except error.RepoError as httpexception:
1115 except error.RepoError as httpexception:
1116 try:
1116 try:
1117 r = statichttprepo.instance(ui, b"static-" + path, create)
1117 r = statichttprepo.instance(ui, b"static-" + path, create)
1118 ui.note(_(b'(falling back to static-http)\n'))
1118 ui.note(_(b'(falling back to static-http)\n'))
1119 return r
1119 return r
1120 except error.RepoError:
1120 except error.RepoError:
1121 raise httpexception # use the original http RepoError instead
1121 raise httpexception # use the original http RepoError instead
General Comments 0
You need to be logged in to leave comments. Login now