##// END OF EJS Templates
httpclient: update to revision 9517a2b56fe9 of httpplus (issue3905)...
Augie Fackler -
r19489:42fcb2f7 stable
parent child Browse files
Show More
@@ -1,691 +1,693
1 1 # Copyright 2010, Google Inc.
2 2 # All rights reserved.
3 3 #
4 4 # Redistribution and use in source and binary forms, with or without
5 5 # modification, are permitted provided that the following conditions are
6 6 # met:
7 7 #
8 8 # * Redistributions of source code must retain the above copyright
9 9 # notice, this list of conditions and the following disclaimer.
10 10 # * Redistributions in binary form must reproduce the above
11 11 # copyright notice, this list of conditions and the following disclaimer
12 12 # in the documentation and/or other materials provided with the
13 13 # distribution.
14 14 # * Neither the name of Google Inc. nor the names of its
15 15 # contributors may be used to endorse or promote products derived from
16 16 # this software without specific prior written permission.
17 17
18 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 29 """Improved HTTP/1.1 client library
30 30
31 31 This library contains an HTTPConnection which is similar to the one in
32 32 httplib, but has several additional features:
33 33
34 34 * supports keepalives natively
35 35 * uses select() to block for incoming data
36 36 * notices when the server responds early to a request
37 37 * implements ssl inline instead of in a different class
38 38 """
39 39
40 40 # Many functions in this file have too many arguments.
41 41 # pylint: disable=R0913
42 42
43 43 import cStringIO
44 44 import errno
45 45 import httplib
46 46 import logging
47 47 import rfc822
48 48 import select
49 49 import socket
50 50
51 51 import _readers
52 52 import socketutil
53 53
54 54 logger = logging.getLogger(__name__)
55 55
56 56 __all__ = ['HTTPConnection', 'HTTPResponse']
57 57
58 58 HTTP_VER_1_0 = 'HTTP/1.0'
59 59 HTTP_VER_1_1 = 'HTTP/1.1'
60 60
61 61 OUTGOING_BUFFER_SIZE = 1 << 15
62 62 INCOMING_BUFFER_SIZE = 1 << 20
63 63
64 64 HDR_ACCEPT_ENCODING = 'accept-encoding'
65 65 HDR_CONNECTION_CTRL = 'connection'
66 66 HDR_CONTENT_LENGTH = 'content-length'
67 67 HDR_XFER_ENCODING = 'transfer-encoding'
68 68
69 69 XFER_ENCODING_CHUNKED = 'chunked'
70 70
71 71 CONNECTION_CLOSE = 'close'
72 72
73 73 EOL = '\r\n'
74 74 _END_HEADERS = EOL * 2
75 75
76 76 # Based on some searching around, 1 second seems like a reasonable
77 77 # default here.
78 78 TIMEOUT_ASSUME_CONTINUE = 1
79 79 TIMEOUT_DEFAULT = None
80 80
81 81
82 82 class HTTPResponse(object):
83 83 """Response from an HTTP server.
84 84
85 85 The response will continue to load as available. If you need the
86 86 complete response before continuing, check the .complete() method.
87 87 """
88 88 def __init__(self, sock, timeout, method):
89 89 self.sock = sock
90 90 self.method = method
91 91 self.raw_response = ''
92 92 self._headers_len = 0
93 93 self.headers = None
94 94 self.will_close = False
95 95 self.status_line = ''
96 96 self.status = None
97 97 self.continued = False
98 98 self.http_version = None
99 99 self.reason = None
100 100 self._reader = None
101 101
102 102 self._read_location = 0
103 103 self._eol = EOL
104 104
105 105 self._timeout = timeout
106 106
107 107 @property
108 108 def _end_headers(self):
109 109 return self._eol * 2
110 110
111 111 def complete(self):
112 112 """Returns true if this response is completely loaded.
113 113
114 114 Note that if this is a connection where complete means the
115 115 socket is closed, this will nearly always return False, even
116 116 in cases where all the data has actually been loaded.
117 117 """
118 118 if self._reader:
119 119 return self._reader.done()
120 120
121 121 def _close(self):
122 122 if self._reader is not None:
123 123 # We're a friend of the reader class here.
124 124 # pylint: disable=W0212
125 125 self._reader._close()
126 126
127 127 def readline(self):
128 128 """Read a single line from the response body.
129 129
130 130 This may block until either a line ending is found or the
131 131 response is complete.
132 132 """
133 133 blocks = []
134 134 while True:
135 135 self._reader.readto('\n', blocks)
136 136
137 137 if blocks and blocks[-1][-1] == '\n' or self.complete():
138 138 break
139 139
140 140 self._select()
141 141
142 142 return ''.join(blocks)
143 143
144 144 def read(self, length=None):
145 145 """Read data from the response body."""
146 146 # if length is None, unbounded read
147 147 while (not self.complete() # never select on a finished read
148 148 and (not length # unbounded, so we wait for complete()
149 149 or length > self._reader.available_data)):
150 150 self._select()
151 151 if not length:
152 152 length = self._reader.available_data
153 153 r = self._reader.read(length)
154 154 if self.complete() and self.will_close:
155 155 self.sock.close()
156 156 return r
157 157
158 158 def _select(self):
159 159 r, unused_write, unused_err = select.select(
160 160 [self.sock], [], [], self._timeout)
161 161 if not r:
162 162 # socket was not readable. If the response is not
163 163 # complete, raise a timeout.
164 164 if not self.complete():
165 165 logger.info('timed out with timeout of %s', self._timeout)
166 166 raise HTTPTimeoutException('timeout reading data')
167 167 try:
168 168 data = self.sock.recv(INCOMING_BUFFER_SIZE)
169 169 except socket.sslerror, e:
170 170 if e.args[0] != socket.SSL_ERROR_WANT_READ:
171 171 raise
172 172 logger.debug('SSL_ERROR_WANT_READ in _select, should retry later')
173 173 return True
174 174 logger.debug('response read %d data during _select', len(data))
175 175 # If the socket was readable and no data was read, that means
176 176 # the socket was closed. Inform the reader (if any) so it can
177 177 # raise an exception if this is an invalid situation.
178 178 if not data:
179 179 if self._reader:
180 180 # We're a friend of the reader class here.
181 181 # pylint: disable=W0212
182 182 self._reader._close()
183 183 return False
184 184 else:
185 185 self._load_response(data)
186 186 return True
187 187
188 188 # This method gets replaced by _load later, which confuses pylint.
189 189 def _load_response(self, data): # pylint: disable=E0202
190 190 # Being here implies we're not at the end of the headers yet,
191 191 # since at the end of this method if headers were completely
192 192 # loaded we replace this method with the load() method of the
193 193 # reader we created.
194 194 self.raw_response += data
195 195 # This is a bogus server with bad line endings
196 196 if self._eol not in self.raw_response:
197 197 for bad_eol in ('\n', '\r'):
198 198 if (bad_eol in self.raw_response
199 199 # verify that bad_eol is not the end of the incoming data
200 200 # as this could be a response line that just got
201 201 # split between \r and \n.
202 202 and (self.raw_response.index(bad_eol) <
203 203 (len(self.raw_response) - 1))):
204 204 logger.info('bogus line endings detected, '
205 205 'using %r for EOL', bad_eol)
206 206 self._eol = bad_eol
207 207 break
208 208 # exit early if not at end of headers
209 209 if self._end_headers not in self.raw_response or self.headers:
210 210 return
211 211
212 212 # handle 100-continue response
213 213 hdrs, body = self.raw_response.split(self._end_headers, 1)
214 214 unused_http_ver, status = hdrs.split(' ', 1)
215 215 if status.startswith('100'):
216 216 self.raw_response = body
217 217 self.continued = True
218 218 logger.debug('continue seen, setting body to %r', body)
219 219 return
220 220
221 221 # arriving here means we should parse response headers
222 222 # as all headers have arrived completely
223 223 hdrs, body = self.raw_response.split(self._end_headers, 1)
224 224 del self.raw_response
225 225 if self._eol in hdrs:
226 226 self.status_line, hdrs = hdrs.split(self._eol, 1)
227 227 else:
228 228 self.status_line = hdrs
229 229 hdrs = ''
230 230 # TODO HTTP < 1.0 support
231 231 (self.http_version, self.status,
232 232 self.reason) = self.status_line.split(' ', 2)
233 233 self.status = int(self.status)
234 234 if self._eol != EOL:
235 235 hdrs = hdrs.replace(self._eol, '\r\n')
236 236 headers = rfc822.Message(cStringIO.StringIO(hdrs))
237 237 content_len = None
238 238 if HDR_CONTENT_LENGTH in headers:
239 239 content_len = int(headers[HDR_CONTENT_LENGTH])
240 240 if self.http_version == HTTP_VER_1_0:
241 241 self.will_close = True
242 242 elif HDR_CONNECTION_CTRL in headers:
243 243 self.will_close = (
244 244 headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
245 245 if (HDR_XFER_ENCODING in headers
246 246 and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
247 247 self._reader = _readers.ChunkedReader(self._eol)
248 248 logger.debug('using a chunked reader')
249 249 else:
250 250 # HEAD responses are forbidden from returning a body, and
251 251 # it's implausible for a CONNECT response to use
252 252 # close-is-end logic for an OK response.
253 253 if (self.method == 'HEAD' or
254 254 (self.method == 'CONNECT' and content_len is None)):
255 255 content_len = 0
256 256 if content_len is not None:
257 257 logger.debug('using a content-length reader with length %d',
258 258 content_len)
259 259 self._reader = _readers.ContentLengthReader(content_len)
260 260 else:
261 261 # Response body had no length specified and is not
262 262 # chunked, so the end of the body will only be
263 263 # identifiable by the termination of the socket by the
264 264 # server. My interpretation of the spec means that we
265 265 # are correct in hitting this case if
266 266 # transfer-encoding, content-length, and
267 267 # connection-control were left unspecified.
268 268 self._reader = _readers.CloseIsEndReader()
269 269 logger.debug('using a close-is-end reader')
270 270 self.will_close = True
271 271
272 272 if body:
273 273 # We're a friend of the reader class here.
274 274 # pylint: disable=W0212
275 275 self._reader._load(body)
276 276 logger.debug('headers complete')
277 277 self.headers = headers
278 278 # We're a friend of the reader class here.
279 279 # pylint: disable=W0212
280 280 self._load_response = self._reader._load
281 281
282 282
283 283 class HTTPConnection(object):
284 284 """Connection to a single http server.
285 285
286 286 Supports 100-continue and keepalives natively. Uses select() for
287 287 non-blocking socket operations.
288 288 """
289 289 http_version = HTTP_VER_1_1
290 290 response_class = HTTPResponse
291 291
292 292 def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
293 293 timeout=TIMEOUT_DEFAULT,
294 294 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
295 295 proxy_hostport=None, **ssl_opts):
296 296 """Create a new HTTPConnection.
297 297
298 298 Args:
299 299 host: The host to which we'll connect.
300 300 port: Optional. The port over which we'll connect. Default 80 for
301 301 non-ssl, 443 for ssl.
302 302 use_ssl: Optional. Whether to use ssl. Defaults to False if port is
303 303 not 443, true if port is 443.
304 304 ssl_validator: a function(socket) to validate the ssl cert
305 305 timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
306 306 continue_timeout: Optional. Timeout for waiting on an expected
307 307 "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
308 308 proxy_hostport: Optional. Tuple of (host, port) to use as an http
309 309 proxy for the connection. Default is to not use a proxy.
310 310 """
311 311 if port is None and host.count(':') == 1 or ']:' in host:
312 312 host, port = host.rsplit(':', 1)
313 313 port = int(port)
314 314 if '[' in host:
315 315 host = host[1:-1]
316 316 if use_ssl is None and port is None:
317 317 use_ssl = False
318 318 port = 80
319 319 elif use_ssl is None:
320 320 use_ssl = (port == 443)
321 321 elif port is None:
322 322 port = (use_ssl and 443 or 80)
323 323 self.port = port
324 324 if use_ssl and not socketutil.have_ssl:
325 325 raise Exception('ssl requested but unavailable on this Python')
326 326 self.ssl = use_ssl
327 327 self.ssl_opts = ssl_opts
328 328 self._ssl_validator = ssl_validator
329 329 self.host = host
330 330 self.sock = None
331 331 self._current_response = None
332 332 self._current_response_taken = False
333 333 if proxy_hostport is None:
334 334 self._proxy_host = self._proxy_port = None
335 335 else:
336 336 self._proxy_host, self._proxy_port = proxy_hostport
337 337
338 338 self.timeout = timeout
339 339 self.continue_timeout = continue_timeout
340 340
341 341 def _connect(self):
342 342 """Connect to the host and port specified in __init__."""
343 343 if self.sock:
344 344 return
345 345 if self._proxy_host is not None:
346 346 logger.info('Connecting to http proxy %s:%s',
347 347 self._proxy_host, self._proxy_port)
348 348 sock = socketutil.create_connection((self._proxy_host,
349 349 self._proxy_port))
350 350 if self.ssl:
351 351 # TODO proxy header support
352 352 data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
353 353 self.port),
354 354 {}, HTTP_VER_1_0)
355 355 sock.send(data)
356 356 sock.setblocking(0)
357 357 r = self.response_class(sock, self.timeout, 'CONNECT')
358 358 timeout_exc = HTTPTimeoutException(
359 359 'Timed out waiting for CONNECT response from proxy')
360 360 while not r.complete():
361 361 try:
362 362 # We're a friend of the response class, so let
363 363 # us use the private attribute.
364 364 # pylint: disable=W0212
365 365 if not r._select():
366 366 if not r.complete():
367 367 raise timeout_exc
368 368 except HTTPTimeoutException:
369 369 # This raise/except pattern looks goofy, but
370 370 # _select can raise the timeout as well as the
371 371 # loop body. I wish it wasn't this convoluted,
372 372 # but I don't have a better solution
373 373 # immediately handy.
374 374 raise timeout_exc
375 375 if r.status != 200:
376 376 raise HTTPProxyConnectFailedException(
377 377 'Proxy connection failed: %d %s' % (r.status,
378 378 r.read()))
379 379 logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
380 380 self.host, self.port)
381 381 else:
382 382 sock = socketutil.create_connection((self.host, self.port))
383 383 if self.ssl:
384 384 # This is the default, but in the case of proxied SSL
385 385 # requests the proxy logic above will have cleared
386 386 # blocking mode, so re-enable it just to be safe.
387 387 sock.setblocking(1)
388 388 logger.debug('wrapping socket for ssl with options %r',
389 389 self.ssl_opts)
390 390 sock = socketutil.wrap_socket(sock, **self.ssl_opts)
391 391 if self._ssl_validator:
392 392 self._ssl_validator(sock)
393 393 sock.setblocking(0)
394 394 self.sock = sock
395 395
396 396 def _buildheaders(self, method, path, headers, http_ver):
397 397 if self.ssl and self.port == 443 or self.port == 80:
398 398 # default port for protocol, so leave it out
399 399 hdrhost = self.host
400 400 else:
401 401 # include nonstandard port in header
402 402 if ':' in self.host: # must be IPv6
403 403 hdrhost = '[%s]:%d' % (self.host, self.port)
404 404 else:
405 405 hdrhost = '%s:%d' % (self.host, self.port)
406 406 if self._proxy_host and not self.ssl:
407 407 # When talking to a regular http proxy we must send the
408 408 # full URI, but in all other cases we must not (although
409 409 # technically RFC 2616 says servers must accept our
410 410 # request if we screw up, experimentally few do that
411 411 # correctly.)
412 412 assert path[0] == '/', 'path must start with a /'
413 413 path = 'http://%s%s' % (hdrhost, path)
414 414 outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
415 415 headers['host'] = ('Host', hdrhost)
416 416 headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
417 417 for hdr, val in headers.itervalues():
418 418 outgoing.append('%s: %s%s' % (hdr, val, EOL))
419 419 outgoing.append(EOL)
420 420 return ''.join(outgoing)
421 421
422 422 def close(self):
423 423 """Close the connection to the server.
424 424
425 425 This is a no-op if the connection is already closed. The
426 426 connection may automatically close if requested by the server
427 427 or required by the nature of a response.
428 428 """
429 429 if self.sock is None:
430 430 return
431 431 self.sock.close()
432 432 self.sock = None
433 433 logger.info('closed connection to %s on %s', self.host, self.port)
434 434
435 435 def busy(self):
436 436 """Returns True if this connection object is currently in use.
437 437
438 438 If a response is still pending, this will return True, even if
439 439 the request has finished sending. In the future,
440 440 HTTPConnection may transparently juggle multiple connections
441 441 to the server, in which case this will be useful to detect if
442 442 any of those connections is ready for use.
443 443 """
444 444 cr = self._current_response
445 445 if cr is not None:
446 446 if self._current_response_taken:
447 447 if cr.will_close:
448 448 self.sock = None
449 449 self._current_response = None
450 450 return False
451 451 elif cr.complete():
452 452 self._current_response = None
453 453 return False
454 454 return True
455 455 return False
456 456
457 457 def _reconnect(self, where):
458 458 logger.info('reconnecting during %s', where)
459 459 self.close()
460 460 self._connect()
461 461
462 462 def request(self, method, path, body=None, headers={},
463 463 expect_continue=False):
464 464 """Send a request to the server.
465 465
466 466 For increased flexibility, this does not return the response
467 467 object. Future versions of HTTPConnection that juggle multiple
468 468 sockets will be able to send (for example) 5 requests all at
469 469 once, and then let the requests arrive as data is
470 470 available. Use the `getresponse()` method to retrieve the
471 471 response.
472 472 """
473 473 if self.busy():
474 474 raise httplib.CannotSendRequest(
475 475 'Can not send another request before '
476 476 'current response is read!')
477 477 self._current_response_taken = False
478 478
479 479 logger.info('sending %s request for %s to %s on port %s',
480 480 method, path, self.host, self.port)
481 481 hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
482 482 if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
483 483 expect_continue = True
484 484 elif expect_continue:
485 485 hdrs['expect'] = ('Expect', '100-Continue')
486 486
487 487 chunked = False
488 488 if body and HDR_CONTENT_LENGTH not in hdrs:
489 489 if getattr(body, '__len__', False):
490 490 hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
491 491 elif getattr(body, 'read', False):
492 492 hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
493 493 XFER_ENCODING_CHUNKED)
494 494 chunked = True
495 495 else:
496 496 raise BadRequestData('body has no __len__() nor read()')
497 497
498 498 self._connect()
499 499 outgoing_headers = self._buildheaders(
500 500 method, path, hdrs, self.http_version)
501 501 response = None
502 502 first = True
503 503
504 504 while ((outgoing_headers or body)
505 505 and not (response and response.complete())):
506 506 select_timeout = self.timeout
507 507 out = outgoing_headers or body
508 508 blocking_on_continue = False
509 509 if expect_continue and not outgoing_headers and not (
510 510 response and (response.headers or response.continued)):
511 511 logger.info(
512 512 'waiting up to %s seconds for'
513 513 ' continue response from server',
514 514 self.continue_timeout)
515 515 select_timeout = self.continue_timeout
516 516 blocking_on_continue = True
517 517 out = False
518 518 if out:
519 519 w = [self.sock]
520 520 else:
521 521 w = []
522 522 r, w, x = select.select([self.sock], w, [], select_timeout)
523 523 # if we were expecting a 100 continue and it's been long
524 524 # enough, just go ahead and assume it's ok. This is the
525 525 # recommended behavior from the RFC.
526 526 if r == w == x == []:
527 527 if blocking_on_continue:
528 528 expect_continue = False
529 529 logger.info('no response to continue expectation from '
530 530 'server, optimistically sending request body')
531 531 else:
532 532 raise HTTPTimeoutException('timeout sending data')
533 533 was_first = first
534 534
535 535 # incoming data
536 536 if r:
537 537 try:
538 538 try:
539 539 data = r[0].recv(INCOMING_BUFFER_SIZE)
540 540 except socket.sslerror, e:
541 541 if e.args[0] != socket.SSL_ERROR_WANT_READ:
542 542 raise
543 543 logger.debug('SSL_ERROR_WANT_READ while sending '
544 544 'data, retrying...')
545 545 continue
546 546 if not data:
547 547 logger.info('socket appears closed in read')
548 548 self.sock = None
549 549 self._current_response = None
550 550 if response is not None:
551 551 # We're a friend of the response class, so let
552 552 # us use the private attribute.
553 553 # pylint: disable=W0212
554 554 response._close()
555 555 # This if/elif ladder is a bit subtle,
556 556 # comments in each branch should help.
557 557 if response is not None and response.complete():
558 558 # Server responded completely and then
559 559 # closed the socket. We should just shut
560 560 # things down and let the caller get their
561 561 # response.
562 562 logger.info('Got an early response, '
563 563 'aborting remaining request.')
564 564 break
565 565 elif was_first and response is None:
566 566 # Most likely a keepalive that got killed
567 567 # on the server's end. Commonly happens
568 568 # after getting a really large response
569 569 # from the server.
570 570 logger.info(
571 571 'Connection appeared closed in read on first'
572 572 ' request loop iteration, will retry.')
573 573 self._reconnect('read')
574 574 continue
575 575 else:
576 576 # We didn't just send the first data hunk,
577 577 # and either have a partial response or no
578 578 # response at all. There's really nothing
579 579 # meaningful we can do here.
580 580 raise HTTPStateError(
581 581 'Connection appears closed after '
582 582 'some request data was written, but the '
583 583 'response was missing or incomplete!')
584 584 logger.debug('read %d bytes in request()', len(data))
585 585 if response is None:
586 586 response = self.response_class(
587 587 r[0], self.timeout, method)
588 588 # We're a friend of the response class, so let us
589 589 # use the private attribute.
590 590 # pylint: disable=W0212
591 591 response._load_response(data)
592 592 # Jump to the next select() call so we load more
593 593 # data if the server is still sending us content.
594 594 continue
595 595 except socket.error, e:
596 596 if e[0] != errno.EPIPE and not was_first:
597 597 raise
598 598
599 599 # outgoing data
600 600 if w and out:
601 601 try:
602 602 if getattr(out, 'read', False):
603 603 # pylint guesses the type of out incorrectly here
604 604 # pylint: disable=E1103
605 605 data = out.read(OUTGOING_BUFFER_SIZE)
606 606 if not data:
607 607 continue
608 608 if len(data) < OUTGOING_BUFFER_SIZE:
609 609 if chunked:
610 610 body = '0' + EOL + EOL
611 611 else:
612 612 body = None
613 613 if chunked:
614 614 out = hex(len(data))[2:] + EOL + data + EOL
615 615 else:
616 616 out = data
617 617 amt = w[0].send(out)
618 618 except socket.error, e:
619 619 if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
620 620 # This means that SSL hasn't flushed its buffer into
621 621 # the socket yet.
622 622 # TODO: find a way to block on ssl flushing its buffer
623 623 # similar to selecting on a raw socket.
624 624 continue
625 if e[0] == errno.EWOULDBLOCK or e[0] == errno.EAGAIN:
626 continue
625 627 elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
626 628 and not first):
627 629 raise
628 630 self._reconnect('write')
629 631 amt = self.sock.send(out)
630 632 logger.debug('sent %d', amt)
631 633 first = False
632 634 if out is body:
633 635 body = out[amt:]
634 636 else:
635 637 outgoing_headers = out[amt:]
636 638
637 639 # close if the server response said to or responded before eating
638 640 # the whole request
639 641 if response is None:
640 642 response = self.response_class(self.sock, self.timeout, method)
641 643 data_left = bool(outgoing_headers or body)
642 644 if data_left:
643 645 logger.info('stopped sending request early, '
644 646 'will close the socket to be safe.')
645 647 response.will_close = True
646 648 if response.will_close:
647 649 # The socket will be closed by the response, so we disown
648 650 # the socket
649 651 self.sock = None
650 652 self._current_response = response
651 653
652 654 def getresponse(self):
653 655 """Returns the response to the most recent request."""
654 656 if self._current_response is None:
655 657 raise httplib.ResponseNotReady()
656 658 r = self._current_response
657 659 while r.headers is None:
658 660 # We're a friend of the response class, so let us use the
659 661 # private attribute.
660 662 # pylint: disable=W0212
661 663 if not r._select() and not r.complete():
662 664 raise _readers.HTTPRemoteClosedError()
663 665 if r.will_close:
664 666 self.sock = None
665 667 self._current_response = None
666 668 elif r.complete():
667 669 self._current_response = None
668 670 else:
669 671 self._current_response_taken = True
670 672 return r
671 673
672 674
673 675 class HTTPTimeoutException(httplib.HTTPException):
674 676 """A timeout occurred while waiting on the server."""
675 677
676 678
677 679 class BadRequestData(httplib.HTTPException):
678 680 """Request body object has neither __len__ nor read."""
679 681
680 682
681 683 class HTTPProxyConnectFailedException(httplib.HTTPException):
682 684 """Connecting to the HTTP proxy failed."""
683 685
684 686
685 687 class HTTPStateError(httplib.HTTPException):
686 688 """Invalid internal state encountered."""
687 689
688 690 # Forward this exception type from _readers since it needs to be part
689 691 # of the public API.
690 692 HTTPRemoteClosedError = _readers.HTTPRemoteClosedError
691 693 # no-check-code
@@ -1,138 +1,138
1 1 # Copyright 2010, Google Inc.
2 2 # All rights reserved.
3 3 #
4 4 # Redistribution and use in source and binary forms, with or without
5 5 # modification, are permitted provided that the following conditions are
6 6 # met:
7 7 #
8 8 # * Redistributions of source code must retain the above copyright
9 9 # notice, this list of conditions and the following disclaimer.
10 10 # * Redistributions in binary form must reproduce the above
11 11 # copyright notice, this list of conditions and the following disclaimer
12 12 # in the documentation and/or other materials provided with the
13 13 # distribution.
14 14 # * Neither the name of Google Inc. nor the names of its
15 15 # contributors may be used to endorse or promote products derived from
16 16 # this software without specific prior written permission.
17 17
18 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 29 """Abstraction to simplify socket use for Python < 2.6
30 30
31 31 This will attempt to use the ssl module and the new
32 32 socket.create_connection method, but fall back to the old
33 33 methods if those are unavailable.
34 34 """
35 35 import logging
36 36 import socket
37 37
38 38 logger = logging.getLogger(__name__)
39 39
40 40 try:
41 41 import ssl
42 42 # make demandimporters load the module
43 43 ssl.wrap_socket # pylint: disable=W0104
44 44 have_ssl = True
45 45 except ImportError:
46 46 import httplib
47 47 import urllib2
48 48 have_ssl = getattr(urllib2, 'HTTPSHandler', False)
49 49 ssl = False
50 50
51 51
52 52 try:
53 53 create_connection = socket.create_connection
54 54 except AttributeError:
55 55 def create_connection(address):
56 56 """Backport of socket.create_connection from Python 2.6."""
57 57 host, port = address
58 58 msg = "getaddrinfo returns an empty list"
59 59 sock = None
60 60 for res in socket.getaddrinfo(host, port, 0,
61 61 socket.SOCK_STREAM):
62 62 af, socktype, proto, unused_canonname, sa = res
63 63 try:
64 64 sock = socket.socket(af, socktype, proto)
65 65 logger.info("connect: (%s, %s)", host, port)
66 66 sock.connect(sa)
67 67 except socket.error, msg:
68 68 logger.info('connect fail: %s %s', host, port)
69 69 if sock:
70 70 sock.close()
71 71 sock = None
72 72 continue
73 73 break
74 74 if not sock:
75 75 raise socket.error(msg)
76 76 return sock
77 77
78 78 if ssl:
79 79 wrap_socket = ssl.wrap_socket
80 80 CERT_NONE = ssl.CERT_NONE
81 81 CERT_OPTIONAL = ssl.CERT_OPTIONAL
82 82 CERT_REQUIRED = ssl.CERT_REQUIRED
83 83 else:
84 84 class FakeSocket(httplib.FakeSocket):
85 85 """Socket wrapper that supports SSL."""
86 86
87 87 # Silence lint about this goofy backport class
88 88 # pylint: disable=W0232,E1101,R0903,R0913,C0111
89 89
90 90 # backport the behavior from Python 2.6, which is to busy wait
91 91 # on the socket instead of anything nice. Sigh.
92 92 # See http://bugs.python.org/issue3890 for more info.
93 93 def recv(self, buflen=1024, flags=0):
94 94 """ssl-aware wrapper around socket.recv
95 95 """
96 96 if flags != 0:
97 97 raise ValueError(
98 98 "non-zero flags not allowed in calls to recv() on %s" %
99 99 self.__class__)
100 100 while True:
101 101 try:
102 102 return self._ssl.read(buflen)
103 103 except socket.sslerror, x:
104 104 if x.args[0] == socket.SSL_ERROR_WANT_READ:
105 105 continue
106 106 else:
107 107 raise x
108 108
109 _PROTOCOL_SSLv23 = 2
109 _PROTOCOL_SSLv3 = 1
110 110
111 111 CERT_NONE = 0
112 112 CERT_OPTIONAL = 1
113 113 CERT_REQUIRED = 2
114 114
115 115 # Disable unused-argument because we're making a dumb wrapper
116 116 # that's like an upstream method.
117 117 #
118 118 # pylint: disable=W0613,R0913
119 119 def wrap_socket(sock, keyfile=None, certfile=None,
120 120 server_side=False, cert_reqs=CERT_NONE,
121 ssl_version=_PROTOCOL_SSLv23, ca_certs=None,
121 ssl_version=_PROTOCOL_SSLv3, ca_certs=None,
122 122 do_handshake_on_connect=True,
123 123 suppress_ragged_eofs=True):
124 124 """Backport of ssl.wrap_socket from Python 2.6."""
125 125 if cert_reqs != CERT_NONE and ca_certs:
126 126 raise CertificateValidationUnsupported(
127 127 'SSL certificate validation requires the ssl module'
128 128 '(included in Python 2.6 and later.)')
129 129 sslob = socket.ssl(sock)
130 130 # borrow httplib's workaround for no ssl.wrap_socket
131 131 sock = FakeSocket(sock, sslob)
132 132 return sock
133 133 # pylint: enable=W0613,R0913
134 134
135 135
136 136 class CertificateValidationUnsupported(Exception):
137 137 """Exception raised when cert validation is requested but unavailable."""
138 138 # no-check-code
General Comments 0
You need to be logged in to leave comments. Login now