##// END OF EJS Templates
httpclient: update to revision 9517a2b56fe9 of httpplus (issue3905)...
Augie Fackler -
r19489:42fcb2f7 stable
parent child Browse files
Show More
@@ -1,691 +1,693
1 # Copyright 2010, Google Inc.
1 # Copyright 2010, Google Inc.
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Redistribution and use in source and binary forms, with or without
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
5 # modification, are permitted provided that the following conditions are
6 # met:
6 # met:
7 #
7 #
8 # * Redistributions of source code must retain the above copyright
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
12 # in the documentation and/or other materials provided with the
13 # distribution.
13 # distribution.
14 # * Neither the name of Google Inc. nor the names of its
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
16 # this software without specific prior written permission.
17
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 """Improved HTTP/1.1 client library
29 """Improved HTTP/1.1 client library
30
30
31 This library contains an HTTPConnection which is similar to the one in
31 This library contains an HTTPConnection which is similar to the one in
32 httplib, but has several additional features:
32 httplib, but has several additional features:
33
33
34 * supports keepalives natively
34 * supports keepalives natively
35 * uses select() to block for incoming data
35 * uses select() to block for incoming data
36 * notices when the server responds early to a request
36 * notices when the server responds early to a request
37 * implements ssl inline instead of in a different class
37 * implements ssl inline instead of in a different class
38 """
38 """
39
39
40 # Many functions in this file have too many arguments.
40 # Many functions in this file have too many arguments.
41 # pylint: disable=R0913
41 # pylint: disable=R0913
42
42
43 import cStringIO
43 import cStringIO
44 import errno
44 import errno
45 import httplib
45 import httplib
46 import logging
46 import logging
47 import rfc822
47 import rfc822
48 import select
48 import select
49 import socket
49 import socket
50
50
51 import _readers
51 import _readers
52 import socketutil
52 import socketutil
53
53
54 logger = logging.getLogger(__name__)
54 logger = logging.getLogger(__name__)
55
55
56 __all__ = ['HTTPConnection', 'HTTPResponse']
56 __all__ = ['HTTPConnection', 'HTTPResponse']
57
57
58 HTTP_VER_1_0 = 'HTTP/1.0'
58 HTTP_VER_1_0 = 'HTTP/1.0'
59 HTTP_VER_1_1 = 'HTTP/1.1'
59 HTTP_VER_1_1 = 'HTTP/1.1'
60
60
61 OUTGOING_BUFFER_SIZE = 1 << 15
61 OUTGOING_BUFFER_SIZE = 1 << 15
62 INCOMING_BUFFER_SIZE = 1 << 20
62 INCOMING_BUFFER_SIZE = 1 << 20
63
63
64 HDR_ACCEPT_ENCODING = 'accept-encoding'
64 HDR_ACCEPT_ENCODING = 'accept-encoding'
65 HDR_CONNECTION_CTRL = 'connection'
65 HDR_CONNECTION_CTRL = 'connection'
66 HDR_CONTENT_LENGTH = 'content-length'
66 HDR_CONTENT_LENGTH = 'content-length'
67 HDR_XFER_ENCODING = 'transfer-encoding'
67 HDR_XFER_ENCODING = 'transfer-encoding'
68
68
69 XFER_ENCODING_CHUNKED = 'chunked'
69 XFER_ENCODING_CHUNKED = 'chunked'
70
70
71 CONNECTION_CLOSE = 'close'
71 CONNECTION_CLOSE = 'close'
72
72
73 EOL = '\r\n'
73 EOL = '\r\n'
74 _END_HEADERS = EOL * 2
74 _END_HEADERS = EOL * 2
75
75
76 # Based on some searching around, 1 second seems like a reasonable
76 # Based on some searching around, 1 second seems like a reasonable
77 # default here.
77 # default here.
78 TIMEOUT_ASSUME_CONTINUE = 1
78 TIMEOUT_ASSUME_CONTINUE = 1
79 TIMEOUT_DEFAULT = None
79 TIMEOUT_DEFAULT = None
80
80
81
81
82 class HTTPResponse(object):
82 class HTTPResponse(object):
83 """Response from an HTTP server.
83 """Response from an HTTP server.
84
84
85 The response will continue to load as available. If you need the
85 The response will continue to load as available. If you need the
86 complete response before continuing, check the .complete() method.
86 complete response before continuing, check the .complete() method.
87 """
87 """
88 def __init__(self, sock, timeout, method):
88 def __init__(self, sock, timeout, method):
89 self.sock = sock
89 self.sock = sock
90 self.method = method
90 self.method = method
91 self.raw_response = ''
91 self.raw_response = ''
92 self._headers_len = 0
92 self._headers_len = 0
93 self.headers = None
93 self.headers = None
94 self.will_close = False
94 self.will_close = False
95 self.status_line = ''
95 self.status_line = ''
96 self.status = None
96 self.status = None
97 self.continued = False
97 self.continued = False
98 self.http_version = None
98 self.http_version = None
99 self.reason = None
99 self.reason = None
100 self._reader = None
100 self._reader = None
101
101
102 self._read_location = 0
102 self._read_location = 0
103 self._eol = EOL
103 self._eol = EOL
104
104
105 self._timeout = timeout
105 self._timeout = timeout
106
106
107 @property
107 @property
108 def _end_headers(self):
108 def _end_headers(self):
109 return self._eol * 2
109 return self._eol * 2
110
110
111 def complete(self):
111 def complete(self):
112 """Returns true if this response is completely loaded.
112 """Returns true if this response is completely loaded.
113
113
114 Note that if this is a connection where complete means the
114 Note that if this is a connection where complete means the
115 socket is closed, this will nearly always return False, even
115 socket is closed, this will nearly always return False, even
116 in cases where all the data has actually been loaded.
116 in cases where all the data has actually been loaded.
117 """
117 """
118 if self._reader:
118 if self._reader:
119 return self._reader.done()
119 return self._reader.done()
120
120
121 def _close(self):
121 def _close(self):
122 if self._reader is not None:
122 if self._reader is not None:
123 # We're a friend of the reader class here.
123 # We're a friend of the reader class here.
124 # pylint: disable=W0212
124 # pylint: disable=W0212
125 self._reader._close()
125 self._reader._close()
126
126
127 def readline(self):
127 def readline(self):
128 """Read a single line from the response body.
128 """Read a single line from the response body.
129
129
130 This may block until either a line ending is found or the
130 This may block until either a line ending is found or the
131 response is complete.
131 response is complete.
132 """
132 """
133 blocks = []
133 blocks = []
134 while True:
134 while True:
135 self._reader.readto('\n', blocks)
135 self._reader.readto('\n', blocks)
136
136
137 if blocks and blocks[-1][-1] == '\n' or self.complete():
137 if blocks and blocks[-1][-1] == '\n' or self.complete():
138 break
138 break
139
139
140 self._select()
140 self._select()
141
141
142 return ''.join(blocks)
142 return ''.join(blocks)
143
143
144 def read(self, length=None):
144 def read(self, length=None):
145 """Read data from the response body."""
145 """Read data from the response body."""
146 # if length is None, unbounded read
146 # if length is None, unbounded read
147 while (not self.complete() # never select on a finished read
147 while (not self.complete() # never select on a finished read
148 and (not length # unbounded, so we wait for complete()
148 and (not length # unbounded, so we wait for complete()
149 or length > self._reader.available_data)):
149 or length > self._reader.available_data)):
150 self._select()
150 self._select()
151 if not length:
151 if not length:
152 length = self._reader.available_data
152 length = self._reader.available_data
153 r = self._reader.read(length)
153 r = self._reader.read(length)
154 if self.complete() and self.will_close:
154 if self.complete() and self.will_close:
155 self.sock.close()
155 self.sock.close()
156 return r
156 return r
157
157
158 def _select(self):
158 def _select(self):
159 r, unused_write, unused_err = select.select(
159 r, unused_write, unused_err = select.select(
160 [self.sock], [], [], self._timeout)
160 [self.sock], [], [], self._timeout)
161 if not r:
161 if not r:
162 # socket was not readable. If the response is not
162 # socket was not readable. If the response is not
163 # complete, raise a timeout.
163 # complete, raise a timeout.
164 if not self.complete():
164 if not self.complete():
165 logger.info('timed out with timeout of %s', self._timeout)
165 logger.info('timed out with timeout of %s', self._timeout)
166 raise HTTPTimeoutException('timeout reading data')
166 raise HTTPTimeoutException('timeout reading data')
167 try:
167 try:
168 data = self.sock.recv(INCOMING_BUFFER_SIZE)
168 data = self.sock.recv(INCOMING_BUFFER_SIZE)
169 except socket.sslerror, e:
169 except socket.sslerror, e:
170 if e.args[0] != socket.SSL_ERROR_WANT_READ:
170 if e.args[0] != socket.SSL_ERROR_WANT_READ:
171 raise
171 raise
172 logger.debug('SSL_ERROR_WANT_READ in _select, should retry later')
172 logger.debug('SSL_ERROR_WANT_READ in _select, should retry later')
173 return True
173 return True
174 logger.debug('response read %d data during _select', len(data))
174 logger.debug('response read %d data during _select', len(data))
175 # If the socket was readable and no data was read, that means
175 # If the socket was readable and no data was read, that means
176 # the socket was closed. Inform the reader (if any) so it can
176 # the socket was closed. Inform the reader (if any) so it can
177 # raise an exception if this is an invalid situation.
177 # raise an exception if this is an invalid situation.
178 if not data:
178 if not data:
179 if self._reader:
179 if self._reader:
180 # We're a friend of the reader class here.
180 # We're a friend of the reader class here.
181 # pylint: disable=W0212
181 # pylint: disable=W0212
182 self._reader._close()
182 self._reader._close()
183 return False
183 return False
184 else:
184 else:
185 self._load_response(data)
185 self._load_response(data)
186 return True
186 return True
187
187
188 # This method gets replaced by _load later, which confuses pylint.
188 # This method gets replaced by _load later, which confuses pylint.
189 def _load_response(self, data): # pylint: disable=E0202
189 def _load_response(self, data): # pylint: disable=E0202
190 # Being here implies we're not at the end of the headers yet,
190 # Being here implies we're not at the end of the headers yet,
191 # since at the end of this method if headers were completely
191 # since at the end of this method if headers were completely
192 # loaded we replace this method with the load() method of the
192 # loaded we replace this method with the load() method of the
193 # reader we created.
193 # reader we created.
194 self.raw_response += data
194 self.raw_response += data
195 # This is a bogus server with bad line endings
195 # This is a bogus server with bad line endings
196 if self._eol not in self.raw_response:
196 if self._eol not in self.raw_response:
197 for bad_eol in ('\n', '\r'):
197 for bad_eol in ('\n', '\r'):
198 if (bad_eol in self.raw_response
198 if (bad_eol in self.raw_response
199 # verify that bad_eol is not the end of the incoming data
199 # verify that bad_eol is not the end of the incoming data
200 # as this could be a response line that just got
200 # as this could be a response line that just got
201 # split between \r and \n.
201 # split between \r and \n.
202 and (self.raw_response.index(bad_eol) <
202 and (self.raw_response.index(bad_eol) <
203 (len(self.raw_response) - 1))):
203 (len(self.raw_response) - 1))):
204 logger.info('bogus line endings detected, '
204 logger.info('bogus line endings detected, '
205 'using %r for EOL', bad_eol)
205 'using %r for EOL', bad_eol)
206 self._eol = bad_eol
206 self._eol = bad_eol
207 break
207 break
208 # exit early if not at end of headers
208 # exit early if not at end of headers
209 if self._end_headers not in self.raw_response or self.headers:
209 if self._end_headers not in self.raw_response or self.headers:
210 return
210 return
211
211
212 # handle 100-continue response
212 # handle 100-continue response
213 hdrs, body = self.raw_response.split(self._end_headers, 1)
213 hdrs, body = self.raw_response.split(self._end_headers, 1)
214 unused_http_ver, status = hdrs.split(' ', 1)
214 unused_http_ver, status = hdrs.split(' ', 1)
215 if status.startswith('100'):
215 if status.startswith('100'):
216 self.raw_response = body
216 self.raw_response = body
217 self.continued = True
217 self.continued = True
218 logger.debug('continue seen, setting body to %r', body)
218 logger.debug('continue seen, setting body to %r', body)
219 return
219 return
220
220
221 # arriving here means we should parse response headers
221 # arriving here means we should parse response headers
222 # as all headers have arrived completely
222 # as all headers have arrived completely
223 hdrs, body = self.raw_response.split(self._end_headers, 1)
223 hdrs, body = self.raw_response.split(self._end_headers, 1)
224 del self.raw_response
224 del self.raw_response
225 if self._eol in hdrs:
225 if self._eol in hdrs:
226 self.status_line, hdrs = hdrs.split(self._eol, 1)
226 self.status_line, hdrs = hdrs.split(self._eol, 1)
227 else:
227 else:
228 self.status_line = hdrs
228 self.status_line = hdrs
229 hdrs = ''
229 hdrs = ''
230 # TODO HTTP < 1.0 support
230 # TODO HTTP < 1.0 support
231 (self.http_version, self.status,
231 (self.http_version, self.status,
232 self.reason) = self.status_line.split(' ', 2)
232 self.reason) = self.status_line.split(' ', 2)
233 self.status = int(self.status)
233 self.status = int(self.status)
234 if self._eol != EOL:
234 if self._eol != EOL:
235 hdrs = hdrs.replace(self._eol, '\r\n')
235 hdrs = hdrs.replace(self._eol, '\r\n')
236 headers = rfc822.Message(cStringIO.StringIO(hdrs))
236 headers = rfc822.Message(cStringIO.StringIO(hdrs))
237 content_len = None
237 content_len = None
238 if HDR_CONTENT_LENGTH in headers:
238 if HDR_CONTENT_LENGTH in headers:
239 content_len = int(headers[HDR_CONTENT_LENGTH])
239 content_len = int(headers[HDR_CONTENT_LENGTH])
240 if self.http_version == HTTP_VER_1_0:
240 if self.http_version == HTTP_VER_1_0:
241 self.will_close = True
241 self.will_close = True
242 elif HDR_CONNECTION_CTRL in headers:
242 elif HDR_CONNECTION_CTRL in headers:
243 self.will_close = (
243 self.will_close = (
244 headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
244 headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
245 if (HDR_XFER_ENCODING in headers
245 if (HDR_XFER_ENCODING in headers
246 and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
246 and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
247 self._reader = _readers.ChunkedReader(self._eol)
247 self._reader = _readers.ChunkedReader(self._eol)
248 logger.debug('using a chunked reader')
248 logger.debug('using a chunked reader')
249 else:
249 else:
250 # HEAD responses are forbidden from returning a body, and
250 # HEAD responses are forbidden from returning a body, and
251 # it's implausible for a CONNECT response to use
251 # it's implausible for a CONNECT response to use
252 # close-is-end logic for an OK response.
252 # close-is-end logic for an OK response.
253 if (self.method == 'HEAD' or
253 if (self.method == 'HEAD' or
254 (self.method == 'CONNECT' and content_len is None)):
254 (self.method == 'CONNECT' and content_len is None)):
255 content_len = 0
255 content_len = 0
256 if content_len is not None:
256 if content_len is not None:
257 logger.debug('using a content-length reader with length %d',
257 logger.debug('using a content-length reader with length %d',
258 content_len)
258 content_len)
259 self._reader = _readers.ContentLengthReader(content_len)
259 self._reader = _readers.ContentLengthReader(content_len)
260 else:
260 else:
261 # Response body had no length specified and is not
261 # Response body had no length specified and is not
262 # chunked, so the end of the body will only be
262 # chunked, so the end of the body will only be
263 # identifiable by the termination of the socket by the
263 # identifiable by the termination of the socket by the
264 # server. My interpretation of the spec means that we
264 # server. My interpretation of the spec means that we
265 # are correct in hitting this case if
265 # are correct in hitting this case if
266 # transfer-encoding, content-length, and
266 # transfer-encoding, content-length, and
267 # connection-control were left unspecified.
267 # connection-control were left unspecified.
268 self._reader = _readers.CloseIsEndReader()
268 self._reader = _readers.CloseIsEndReader()
269 logger.debug('using a close-is-end reader')
269 logger.debug('using a close-is-end reader')
270 self.will_close = True
270 self.will_close = True
271
271
272 if body:
272 if body:
273 # We're a friend of the reader class here.
273 # We're a friend of the reader class here.
274 # pylint: disable=W0212
274 # pylint: disable=W0212
275 self._reader._load(body)
275 self._reader._load(body)
276 logger.debug('headers complete')
276 logger.debug('headers complete')
277 self.headers = headers
277 self.headers = headers
278 # We're a friend of the reader class here.
278 # We're a friend of the reader class here.
279 # pylint: disable=W0212
279 # pylint: disable=W0212
280 self._load_response = self._reader._load
280 self._load_response = self._reader._load
281
281
282
282
283 class HTTPConnection(object):
283 class HTTPConnection(object):
284 """Connection to a single http server.
284 """Connection to a single http server.
285
285
286 Supports 100-continue and keepalives natively. Uses select() for
286 Supports 100-continue and keepalives natively. Uses select() for
287 non-blocking socket operations.
287 non-blocking socket operations.
288 """
288 """
289 http_version = HTTP_VER_1_1
289 http_version = HTTP_VER_1_1
290 response_class = HTTPResponse
290 response_class = HTTPResponse
291
291
292 def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
292 def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
293 timeout=TIMEOUT_DEFAULT,
293 timeout=TIMEOUT_DEFAULT,
294 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
294 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
295 proxy_hostport=None, **ssl_opts):
295 proxy_hostport=None, **ssl_opts):
296 """Create a new HTTPConnection.
296 """Create a new HTTPConnection.
297
297
298 Args:
298 Args:
299 host: The host to which we'll connect.
299 host: The host to which we'll connect.
300 port: Optional. The port over which we'll connect. Default 80 for
300 port: Optional. The port over which we'll connect. Default 80 for
301 non-ssl, 443 for ssl.
301 non-ssl, 443 for ssl.
302 use_ssl: Optional. Whether to use ssl. Defaults to False if port is
302 use_ssl: Optional. Whether to use ssl. Defaults to False if port is
303 not 443, true if port is 443.
303 not 443, true if port is 443.
304 ssl_validator: a function(socket) to validate the ssl cert
304 ssl_validator: a function(socket) to validate the ssl cert
305 timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
305 timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
306 continue_timeout: Optional. Timeout for waiting on an expected
306 continue_timeout: Optional. Timeout for waiting on an expected
307 "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
307 "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
308 proxy_hostport: Optional. Tuple of (host, port) to use as an http
308 proxy_hostport: Optional. Tuple of (host, port) to use as an http
309 proxy for the connection. Default is to not use a proxy.
309 proxy for the connection. Default is to not use a proxy.
310 """
310 """
311 if port is None and host.count(':') == 1 or ']:' in host:
311 if port is None and host.count(':') == 1 or ']:' in host:
312 host, port = host.rsplit(':', 1)
312 host, port = host.rsplit(':', 1)
313 port = int(port)
313 port = int(port)
314 if '[' in host:
314 if '[' in host:
315 host = host[1:-1]
315 host = host[1:-1]
316 if use_ssl is None and port is None:
316 if use_ssl is None and port is None:
317 use_ssl = False
317 use_ssl = False
318 port = 80
318 port = 80
319 elif use_ssl is None:
319 elif use_ssl is None:
320 use_ssl = (port == 443)
320 use_ssl = (port == 443)
321 elif port is None:
321 elif port is None:
322 port = (use_ssl and 443 or 80)
322 port = (use_ssl and 443 or 80)
323 self.port = port
323 self.port = port
324 if use_ssl and not socketutil.have_ssl:
324 if use_ssl and not socketutil.have_ssl:
325 raise Exception('ssl requested but unavailable on this Python')
325 raise Exception('ssl requested but unavailable on this Python')
326 self.ssl = use_ssl
326 self.ssl = use_ssl
327 self.ssl_opts = ssl_opts
327 self.ssl_opts = ssl_opts
328 self._ssl_validator = ssl_validator
328 self._ssl_validator = ssl_validator
329 self.host = host
329 self.host = host
330 self.sock = None
330 self.sock = None
331 self._current_response = None
331 self._current_response = None
332 self._current_response_taken = False
332 self._current_response_taken = False
333 if proxy_hostport is None:
333 if proxy_hostport is None:
334 self._proxy_host = self._proxy_port = None
334 self._proxy_host = self._proxy_port = None
335 else:
335 else:
336 self._proxy_host, self._proxy_port = proxy_hostport
336 self._proxy_host, self._proxy_port = proxy_hostport
337
337
338 self.timeout = timeout
338 self.timeout = timeout
339 self.continue_timeout = continue_timeout
339 self.continue_timeout = continue_timeout
340
340
341 def _connect(self):
341 def _connect(self):
342 """Connect to the host and port specified in __init__."""
342 """Connect to the host and port specified in __init__."""
343 if self.sock:
343 if self.sock:
344 return
344 return
345 if self._proxy_host is not None:
345 if self._proxy_host is not None:
346 logger.info('Connecting to http proxy %s:%s',
346 logger.info('Connecting to http proxy %s:%s',
347 self._proxy_host, self._proxy_port)
347 self._proxy_host, self._proxy_port)
348 sock = socketutil.create_connection((self._proxy_host,
348 sock = socketutil.create_connection((self._proxy_host,
349 self._proxy_port))
349 self._proxy_port))
350 if self.ssl:
350 if self.ssl:
351 # TODO proxy header support
351 # TODO proxy header support
352 data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
352 data = self._buildheaders('CONNECT', '%s:%d' % (self.host,
353 self.port),
353 self.port),
354 {}, HTTP_VER_1_0)
354 {}, HTTP_VER_1_0)
355 sock.send(data)
355 sock.send(data)
356 sock.setblocking(0)
356 sock.setblocking(0)
357 r = self.response_class(sock, self.timeout, 'CONNECT')
357 r = self.response_class(sock, self.timeout, 'CONNECT')
358 timeout_exc = HTTPTimeoutException(
358 timeout_exc = HTTPTimeoutException(
359 'Timed out waiting for CONNECT response from proxy')
359 'Timed out waiting for CONNECT response from proxy')
360 while not r.complete():
360 while not r.complete():
361 try:
361 try:
362 # We're a friend of the response class, so let
362 # We're a friend of the response class, so let
363 # us use the private attribute.
363 # us use the private attribute.
364 # pylint: disable=W0212
364 # pylint: disable=W0212
365 if not r._select():
365 if not r._select():
366 if not r.complete():
366 if not r.complete():
367 raise timeout_exc
367 raise timeout_exc
368 except HTTPTimeoutException:
368 except HTTPTimeoutException:
369 # This raise/except pattern looks goofy, but
369 # This raise/except pattern looks goofy, but
370 # _select can raise the timeout as well as the
370 # _select can raise the timeout as well as the
371 # loop body. I wish it wasn't this convoluted,
371 # loop body. I wish it wasn't this convoluted,
372 # but I don't have a better solution
372 # but I don't have a better solution
373 # immediately handy.
373 # immediately handy.
374 raise timeout_exc
374 raise timeout_exc
375 if r.status != 200:
375 if r.status != 200:
376 raise HTTPProxyConnectFailedException(
376 raise HTTPProxyConnectFailedException(
377 'Proxy connection failed: %d %s' % (r.status,
377 'Proxy connection failed: %d %s' % (r.status,
378 r.read()))
378 r.read()))
379 logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
379 logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
380 self.host, self.port)
380 self.host, self.port)
381 else:
381 else:
382 sock = socketutil.create_connection((self.host, self.port))
382 sock = socketutil.create_connection((self.host, self.port))
383 if self.ssl:
383 if self.ssl:
384 # This is the default, but in the case of proxied SSL
384 # This is the default, but in the case of proxied SSL
385 # requests the proxy logic above will have cleared
385 # requests the proxy logic above will have cleared
386 # blocking mode, so re-enable it just to be safe.
386 # blocking mode, so re-enable it just to be safe.
387 sock.setblocking(1)
387 sock.setblocking(1)
388 logger.debug('wrapping socket for ssl with options %r',
388 logger.debug('wrapping socket for ssl with options %r',
389 self.ssl_opts)
389 self.ssl_opts)
390 sock = socketutil.wrap_socket(sock, **self.ssl_opts)
390 sock = socketutil.wrap_socket(sock, **self.ssl_opts)
391 if self._ssl_validator:
391 if self._ssl_validator:
392 self._ssl_validator(sock)
392 self._ssl_validator(sock)
393 sock.setblocking(0)
393 sock.setblocking(0)
394 self.sock = sock
394 self.sock = sock
395
395
396 def _buildheaders(self, method, path, headers, http_ver):
396 def _buildheaders(self, method, path, headers, http_ver):
397 if self.ssl and self.port == 443 or self.port == 80:
397 if self.ssl and self.port == 443 or self.port == 80:
398 # default port for protocol, so leave it out
398 # default port for protocol, so leave it out
399 hdrhost = self.host
399 hdrhost = self.host
400 else:
400 else:
401 # include nonstandard port in header
401 # include nonstandard port in header
402 if ':' in self.host: # must be IPv6
402 if ':' in self.host: # must be IPv6
403 hdrhost = '[%s]:%d' % (self.host, self.port)
403 hdrhost = '[%s]:%d' % (self.host, self.port)
404 else:
404 else:
405 hdrhost = '%s:%d' % (self.host, self.port)
405 hdrhost = '%s:%d' % (self.host, self.port)
406 if self._proxy_host and not self.ssl:
406 if self._proxy_host and not self.ssl:
407 # When talking to a regular http proxy we must send the
407 # When talking to a regular http proxy we must send the
408 # full URI, but in all other cases we must not (although
408 # full URI, but in all other cases we must not (although
409 # technically RFC 2616 says servers must accept our
409 # technically RFC 2616 says servers must accept our
410 # request if we screw up, experimentally few do that
410 # request if we screw up, experimentally few do that
411 # correctly.)
411 # correctly.)
412 assert path[0] == '/', 'path must start with a /'
412 assert path[0] == '/', 'path must start with a /'
413 path = 'http://%s%s' % (hdrhost, path)
413 path = 'http://%s%s' % (hdrhost, path)
414 outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
414 outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
415 headers['host'] = ('Host', hdrhost)
415 headers['host'] = ('Host', hdrhost)
416 headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
416 headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
417 for hdr, val in headers.itervalues():
417 for hdr, val in headers.itervalues():
418 outgoing.append('%s: %s%s' % (hdr, val, EOL))
418 outgoing.append('%s: %s%s' % (hdr, val, EOL))
419 outgoing.append(EOL)
419 outgoing.append(EOL)
420 return ''.join(outgoing)
420 return ''.join(outgoing)
421
421
422 def close(self):
422 def close(self):
423 """Close the connection to the server.
423 """Close the connection to the server.
424
424
425 This is a no-op if the connection is already closed. The
425 This is a no-op if the connection is already closed. The
426 connection may automatically close if requested by the server
426 connection may automatically close if requested by the server
427 or required by the nature of a response.
427 or required by the nature of a response.
428 """
428 """
429 if self.sock is None:
429 if self.sock is None:
430 return
430 return
431 self.sock.close()
431 self.sock.close()
432 self.sock = None
432 self.sock = None
433 logger.info('closed connection to %s on %s', self.host, self.port)
433 logger.info('closed connection to %s on %s', self.host, self.port)
434
434
435 def busy(self):
435 def busy(self):
436 """Returns True if this connection object is currently in use.
436 """Returns True if this connection object is currently in use.
437
437
438 If a response is still pending, this will return True, even if
438 If a response is still pending, this will return True, even if
439 the request has finished sending. In the future,
439 the request has finished sending. In the future,
440 HTTPConnection may transparently juggle multiple connections
440 HTTPConnection may transparently juggle multiple connections
441 to the server, in which case this will be useful to detect if
441 to the server, in which case this will be useful to detect if
442 any of those connections is ready for use.
442 any of those connections is ready for use.
443 """
443 """
444 cr = self._current_response
444 cr = self._current_response
445 if cr is not None:
445 if cr is not None:
446 if self._current_response_taken:
446 if self._current_response_taken:
447 if cr.will_close:
447 if cr.will_close:
448 self.sock = None
448 self.sock = None
449 self._current_response = None
449 self._current_response = None
450 return False
450 return False
451 elif cr.complete():
451 elif cr.complete():
452 self._current_response = None
452 self._current_response = None
453 return False
453 return False
454 return True
454 return True
455 return False
455 return False
456
456
457 def _reconnect(self, where):
457 def _reconnect(self, where):
458 logger.info('reconnecting during %s', where)
458 logger.info('reconnecting during %s', where)
459 self.close()
459 self.close()
460 self._connect()
460 self._connect()
461
461
462 def request(self, method, path, body=None, headers={},
462 def request(self, method, path, body=None, headers={},
463 expect_continue=False):
463 expect_continue=False):
464 """Send a request to the server.
464 """Send a request to the server.
465
465
466 For increased flexibility, this does not return the response
466 For increased flexibility, this does not return the response
467 object. Future versions of HTTPConnection that juggle multiple
467 object. Future versions of HTTPConnection that juggle multiple
468 sockets will be able to send (for example) 5 requests all at
468 sockets will be able to send (for example) 5 requests all at
469 once, and then let the requests arrive as data is
469 once, and then let the requests arrive as data is
470 available. Use the `getresponse()` method to retrieve the
470 available. Use the `getresponse()` method to retrieve the
471 response.
471 response.
472 """
472 """
473 if self.busy():
473 if self.busy():
474 raise httplib.CannotSendRequest(
474 raise httplib.CannotSendRequest(
475 'Can not send another request before '
475 'Can not send another request before '
476 'current response is read!')
476 'current response is read!')
477 self._current_response_taken = False
477 self._current_response_taken = False
478
478
479 logger.info('sending %s request for %s to %s on port %s',
479 logger.info('sending %s request for %s to %s on port %s',
480 method, path, self.host, self.port)
480 method, path, self.host, self.port)
481 hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
481 hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
482 if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
482 if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
483 expect_continue = True
483 expect_continue = True
484 elif expect_continue:
484 elif expect_continue:
485 hdrs['expect'] = ('Expect', '100-Continue')
485 hdrs['expect'] = ('Expect', '100-Continue')
486
486
487 chunked = False
487 chunked = False
488 if body and HDR_CONTENT_LENGTH not in hdrs:
488 if body and HDR_CONTENT_LENGTH not in hdrs:
489 if getattr(body, '__len__', False):
489 if getattr(body, '__len__', False):
490 hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
490 hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
491 elif getattr(body, 'read', False):
491 elif getattr(body, 'read', False):
492 hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
492 hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
493 XFER_ENCODING_CHUNKED)
493 XFER_ENCODING_CHUNKED)
494 chunked = True
494 chunked = True
495 else:
495 else:
496 raise BadRequestData('body has no __len__() nor read()')
496 raise BadRequestData('body has no __len__() nor read()')
497
497
498 self._connect()
498 self._connect()
499 outgoing_headers = self._buildheaders(
499 outgoing_headers = self._buildheaders(
500 method, path, hdrs, self.http_version)
500 method, path, hdrs, self.http_version)
501 response = None
501 response = None
502 first = True
502 first = True
503
503
504 while ((outgoing_headers or body)
504 while ((outgoing_headers or body)
505 and not (response and response.complete())):
505 and not (response and response.complete())):
506 select_timeout = self.timeout
506 select_timeout = self.timeout
507 out = outgoing_headers or body
507 out = outgoing_headers or body
508 blocking_on_continue = False
508 blocking_on_continue = False
509 if expect_continue and not outgoing_headers and not (
509 if expect_continue and not outgoing_headers and not (
510 response and (response.headers or response.continued)):
510 response and (response.headers or response.continued)):
511 logger.info(
511 logger.info(
512 'waiting up to %s seconds for'
512 'waiting up to %s seconds for'
513 ' continue response from server',
513 ' continue response from server',
514 self.continue_timeout)
514 self.continue_timeout)
515 select_timeout = self.continue_timeout
515 select_timeout = self.continue_timeout
516 blocking_on_continue = True
516 blocking_on_continue = True
517 out = False
517 out = False
518 if out:
518 if out:
519 w = [self.sock]
519 w = [self.sock]
520 else:
520 else:
521 w = []
521 w = []
522 r, w, x = select.select([self.sock], w, [], select_timeout)
522 r, w, x = select.select([self.sock], w, [], select_timeout)
523 # if we were expecting a 100 continue and it's been long
523 # if we were expecting a 100 continue and it's been long
524 # enough, just go ahead and assume it's ok. This is the
524 # enough, just go ahead and assume it's ok. This is the
525 # recommended behavior from the RFC.
525 # recommended behavior from the RFC.
526 if r == w == x == []:
526 if r == w == x == []:
527 if blocking_on_continue:
527 if blocking_on_continue:
528 expect_continue = False
528 expect_continue = False
529 logger.info('no response to continue expectation from '
529 logger.info('no response to continue expectation from '
530 'server, optimistically sending request body')
530 'server, optimistically sending request body')
531 else:
531 else:
532 raise HTTPTimeoutException('timeout sending data')
532 raise HTTPTimeoutException('timeout sending data')
533 was_first = first
533 was_first = first
534
534
535 # incoming data
535 # incoming data
536 if r:
536 if r:
537 try:
537 try:
538 try:
538 try:
539 data = r[0].recv(INCOMING_BUFFER_SIZE)
539 data = r[0].recv(INCOMING_BUFFER_SIZE)
540 except socket.sslerror, e:
540 except socket.sslerror, e:
541 if e.args[0] != socket.SSL_ERROR_WANT_READ:
541 if e.args[0] != socket.SSL_ERROR_WANT_READ:
542 raise
542 raise
543 logger.debug('SSL_ERROR_WANT_READ while sending '
543 logger.debug('SSL_ERROR_WANT_READ while sending '
544 'data, retrying...')
544 'data, retrying...')
545 continue
545 continue
546 if not data:
546 if not data:
547 logger.info('socket appears closed in read')
547 logger.info('socket appears closed in read')
548 self.sock = None
548 self.sock = None
549 self._current_response = None
549 self._current_response = None
550 if response is not None:
550 if response is not None:
551 # We're a friend of the response class, so let
551 # We're a friend of the response class, so let
552 # us use the private attribute.
552 # us use the private attribute.
553 # pylint: disable=W0212
553 # pylint: disable=W0212
554 response._close()
554 response._close()
555 # This if/elif ladder is a bit subtle,
555 # This if/elif ladder is a bit subtle,
556 # comments in each branch should help.
556 # comments in each branch should help.
557 if response is not None and response.complete():
557 if response is not None and response.complete():
558 # Server responded completely and then
558 # Server responded completely and then
559 # closed the socket. We should just shut
559 # closed the socket. We should just shut
560 # things down and let the caller get their
560 # things down and let the caller get their
561 # response.
561 # response.
562 logger.info('Got an early response, '
562 logger.info('Got an early response, '
563 'aborting remaining request.')
563 'aborting remaining request.')
564 break
564 break
565 elif was_first and response is None:
565 elif was_first and response is None:
566 # Most likely a keepalive that got killed
566 # Most likely a keepalive that got killed
567 # on the server's end. Commonly happens
567 # on the server's end. Commonly happens
568 # after getting a really large response
568 # after getting a really large response
569 # from the server.
569 # from the server.
570 logger.info(
570 logger.info(
571 'Connection appeared closed in read on first'
571 'Connection appeared closed in read on first'
572 ' request loop iteration, will retry.')
572 ' request loop iteration, will retry.')
573 self._reconnect('read')
573 self._reconnect('read')
574 continue
574 continue
575 else:
575 else:
576 # We didn't just send the first data hunk,
576 # We didn't just send the first data hunk,
577 # and either have a partial response or no
577 # and either have a partial response or no
578 # response at all. There's really nothing
578 # response at all. There's really nothing
579 # meaningful we can do here.
579 # meaningful we can do here.
580 raise HTTPStateError(
580 raise HTTPStateError(
581 'Connection appears closed after '
581 'Connection appears closed after '
582 'some request data was written, but the '
582 'some request data was written, but the '
583 'response was missing or incomplete!')
583 'response was missing or incomplete!')
584 logger.debug('read %d bytes in request()', len(data))
584 logger.debug('read %d bytes in request()', len(data))
585 if response is None:
585 if response is None:
586 response = self.response_class(
586 response = self.response_class(
587 r[0], self.timeout, method)
587 r[0], self.timeout, method)
588 # We're a friend of the response class, so let us
588 # We're a friend of the response class, so let us
589 # use the private attribute.
589 # use the private attribute.
590 # pylint: disable=W0212
590 # pylint: disable=W0212
591 response._load_response(data)
591 response._load_response(data)
592 # Jump to the next select() call so we load more
592 # Jump to the next select() call so we load more
593 # data if the server is still sending us content.
593 # data if the server is still sending us content.
594 continue
594 continue
595 except socket.error, e:
595 except socket.error, e:
596 if e[0] != errno.EPIPE and not was_first:
596 if e[0] != errno.EPIPE and not was_first:
597 raise
597 raise
598
598
599 # outgoing data
599 # outgoing data
600 if w and out:
600 if w and out:
601 try:
601 try:
602 if getattr(out, 'read', False):
602 if getattr(out, 'read', False):
603 # pylint guesses the type of out incorrectly here
603 # pylint guesses the type of out incorrectly here
604 # pylint: disable=E1103
604 # pylint: disable=E1103
605 data = out.read(OUTGOING_BUFFER_SIZE)
605 data = out.read(OUTGOING_BUFFER_SIZE)
606 if not data:
606 if not data:
607 continue
607 continue
608 if len(data) < OUTGOING_BUFFER_SIZE:
608 if len(data) < OUTGOING_BUFFER_SIZE:
609 if chunked:
609 if chunked:
610 body = '0' + EOL + EOL
610 body = '0' + EOL + EOL
611 else:
611 else:
612 body = None
612 body = None
613 if chunked:
613 if chunked:
614 out = hex(len(data))[2:] + EOL + data + EOL
614 out = hex(len(data))[2:] + EOL + data + EOL
615 else:
615 else:
616 out = data
616 out = data
617 amt = w[0].send(out)
617 amt = w[0].send(out)
618 except socket.error, e:
618 except socket.error, e:
619 if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
619 if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
620 # This means that SSL hasn't flushed its buffer into
620 # This means that SSL hasn't flushed its buffer into
621 # the socket yet.
621 # the socket yet.
622 # TODO: find a way to block on ssl flushing its buffer
622 # TODO: find a way to block on ssl flushing its buffer
623 # similar to selecting on a raw socket.
623 # similar to selecting on a raw socket.
624 continue
624 continue
625 if e[0] == errno.EWOULDBLOCK or e[0] == errno.EAGAIN:
626 continue
625 elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
627 elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
626 and not first):
628 and not first):
627 raise
629 raise
628 self._reconnect('write')
630 self._reconnect('write')
629 amt = self.sock.send(out)
631 amt = self.sock.send(out)
630 logger.debug('sent %d', amt)
632 logger.debug('sent %d', amt)
631 first = False
633 first = False
632 if out is body:
634 if out is body:
633 body = out[amt:]
635 body = out[amt:]
634 else:
636 else:
635 outgoing_headers = out[amt:]
637 outgoing_headers = out[amt:]
636
638
637 # close if the server response said to or responded before eating
639 # close if the server response said to or responded before eating
638 # the whole request
640 # the whole request
639 if response is None:
641 if response is None:
640 response = self.response_class(self.sock, self.timeout, method)
642 response = self.response_class(self.sock, self.timeout, method)
641 data_left = bool(outgoing_headers or body)
643 data_left = bool(outgoing_headers or body)
642 if data_left:
644 if data_left:
643 logger.info('stopped sending request early, '
645 logger.info('stopped sending request early, '
644 'will close the socket to be safe.')
646 'will close the socket to be safe.')
645 response.will_close = True
647 response.will_close = True
646 if response.will_close:
648 if response.will_close:
647 # The socket will be closed by the response, so we disown
649 # The socket will be closed by the response, so we disown
648 # the socket
650 # the socket
649 self.sock = None
651 self.sock = None
650 self._current_response = response
652 self._current_response = response
651
653
652 def getresponse(self):
654 def getresponse(self):
653 """Returns the response to the most recent request."""
655 """Returns the response to the most recent request."""
654 if self._current_response is None:
656 if self._current_response is None:
655 raise httplib.ResponseNotReady()
657 raise httplib.ResponseNotReady()
656 r = self._current_response
658 r = self._current_response
657 while r.headers is None:
659 while r.headers is None:
658 # We're a friend of the response class, so let us use the
660 # We're a friend of the response class, so let us use the
659 # private attribute.
661 # private attribute.
660 # pylint: disable=W0212
662 # pylint: disable=W0212
661 if not r._select() and not r.complete():
663 if not r._select() and not r.complete():
662 raise _readers.HTTPRemoteClosedError()
664 raise _readers.HTTPRemoteClosedError()
663 if r.will_close:
665 if r.will_close:
664 self.sock = None
666 self.sock = None
665 self._current_response = None
667 self._current_response = None
666 elif r.complete():
668 elif r.complete():
667 self._current_response = None
669 self._current_response = None
668 else:
670 else:
669 self._current_response_taken = True
671 self._current_response_taken = True
670 return r
672 return r
671
673
672
674
673 class HTTPTimeoutException(httplib.HTTPException):
675 class HTTPTimeoutException(httplib.HTTPException):
674 """A timeout occurred while waiting on the server."""
676 """A timeout occurred while waiting on the server."""
675
677
676
678
677 class BadRequestData(httplib.HTTPException):
679 class BadRequestData(httplib.HTTPException):
678 """Request body object has neither __len__ nor read."""
680 """Request body object has neither __len__ nor read."""
679
681
680
682
681 class HTTPProxyConnectFailedException(httplib.HTTPException):
683 class HTTPProxyConnectFailedException(httplib.HTTPException):
682 """Connecting to the HTTP proxy failed."""
684 """Connecting to the HTTP proxy failed."""
683
685
684
686
685 class HTTPStateError(httplib.HTTPException):
687 class HTTPStateError(httplib.HTTPException):
686 """Invalid internal state encountered."""
688 """Invalid internal state encountered."""
687
689
688 # Forward this exception type from _readers since it needs to be part
690 # Forward this exception type from _readers since it needs to be part
689 # of the public API.
691 # of the public API.
690 HTTPRemoteClosedError = _readers.HTTPRemoteClosedError
692 HTTPRemoteClosedError = _readers.HTTPRemoteClosedError
691 # no-check-code
693 # no-check-code
@@ -1,138 +1,138
1 # Copyright 2010, Google Inc.
1 # Copyright 2010, Google Inc.
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Redistribution and use in source and binary forms, with or without
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
5 # modification, are permitted provided that the following conditions are
6 # met:
6 # met:
7 #
7 #
8 # * Redistributions of source code must retain the above copyright
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
12 # in the documentation and/or other materials provided with the
13 # distribution.
13 # distribution.
14 # * Neither the name of Google Inc. nor the names of its
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
16 # this software without specific prior written permission.
17
17
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 """Abstraction to simplify socket use for Python < 2.6
29 """Abstraction to simplify socket use for Python < 2.6
30
30
31 This will attempt to use the ssl module and the new
31 This will attempt to use the ssl module and the new
32 socket.create_connection method, but fall back to the old
32 socket.create_connection method, but fall back to the old
33 methods if those are unavailable.
33 methods if those are unavailable.
34 """
34 """
35 import logging
35 import logging
36 import socket
36 import socket
37
37
38 logger = logging.getLogger(__name__)
38 logger = logging.getLogger(__name__)
39
39
40 try:
40 try:
41 import ssl
41 import ssl
42 # make demandimporters load the module
42 # make demandimporters load the module
43 ssl.wrap_socket # pylint: disable=W0104
43 ssl.wrap_socket # pylint: disable=W0104
44 have_ssl = True
44 have_ssl = True
45 except ImportError:
45 except ImportError:
46 import httplib
46 import httplib
47 import urllib2
47 import urllib2
48 have_ssl = getattr(urllib2, 'HTTPSHandler', False)
48 have_ssl = getattr(urllib2, 'HTTPSHandler', False)
49 ssl = False
49 ssl = False
50
50
51
51
52 try:
52 try:
53 create_connection = socket.create_connection
53 create_connection = socket.create_connection
54 except AttributeError:
54 except AttributeError:
55 def create_connection(address):
55 def create_connection(address):
56 """Backport of socket.create_connection from Python 2.6."""
56 """Backport of socket.create_connection from Python 2.6."""
57 host, port = address
57 host, port = address
58 msg = "getaddrinfo returns an empty list"
58 msg = "getaddrinfo returns an empty list"
59 sock = None
59 sock = None
60 for res in socket.getaddrinfo(host, port, 0,
60 for res in socket.getaddrinfo(host, port, 0,
61 socket.SOCK_STREAM):
61 socket.SOCK_STREAM):
62 af, socktype, proto, unused_canonname, sa = res
62 af, socktype, proto, unused_canonname, sa = res
63 try:
63 try:
64 sock = socket.socket(af, socktype, proto)
64 sock = socket.socket(af, socktype, proto)
65 logger.info("connect: (%s, %s)", host, port)
65 logger.info("connect: (%s, %s)", host, port)
66 sock.connect(sa)
66 sock.connect(sa)
67 except socket.error, msg:
67 except socket.error, msg:
68 logger.info('connect fail: %s %s', host, port)
68 logger.info('connect fail: %s %s', host, port)
69 if sock:
69 if sock:
70 sock.close()
70 sock.close()
71 sock = None
71 sock = None
72 continue
72 continue
73 break
73 break
74 if not sock:
74 if not sock:
75 raise socket.error(msg)
75 raise socket.error(msg)
76 return sock
76 return sock
77
77
78 if ssl:
78 if ssl:
79 wrap_socket = ssl.wrap_socket
79 wrap_socket = ssl.wrap_socket
80 CERT_NONE = ssl.CERT_NONE
80 CERT_NONE = ssl.CERT_NONE
81 CERT_OPTIONAL = ssl.CERT_OPTIONAL
81 CERT_OPTIONAL = ssl.CERT_OPTIONAL
82 CERT_REQUIRED = ssl.CERT_REQUIRED
82 CERT_REQUIRED = ssl.CERT_REQUIRED
83 else:
83 else:
84 class FakeSocket(httplib.FakeSocket):
84 class FakeSocket(httplib.FakeSocket):
85 """Socket wrapper that supports SSL."""
85 """Socket wrapper that supports SSL."""
86
86
87 # Silence lint about this goofy backport class
87 # Silence lint about this goofy backport class
88 # pylint: disable=W0232,E1101,R0903,R0913,C0111
88 # pylint: disable=W0232,E1101,R0903,R0913,C0111
89
89
90 # backport the behavior from Python 2.6, which is to busy wait
90 # backport the behavior from Python 2.6, which is to busy wait
91 # on the socket instead of anything nice. Sigh.
91 # on the socket instead of anything nice. Sigh.
92 # See http://bugs.python.org/issue3890 for more info.
92 # See http://bugs.python.org/issue3890 for more info.
93 def recv(self, buflen=1024, flags=0):
93 def recv(self, buflen=1024, flags=0):
94 """ssl-aware wrapper around socket.recv
94 """ssl-aware wrapper around socket.recv
95 """
95 """
96 if flags != 0:
96 if flags != 0:
97 raise ValueError(
97 raise ValueError(
98 "non-zero flags not allowed in calls to recv() on %s" %
98 "non-zero flags not allowed in calls to recv() on %s" %
99 self.__class__)
99 self.__class__)
100 while True:
100 while True:
101 try:
101 try:
102 return self._ssl.read(buflen)
102 return self._ssl.read(buflen)
103 except socket.sslerror, x:
103 except socket.sslerror, x:
104 if x.args[0] == socket.SSL_ERROR_WANT_READ:
104 if x.args[0] == socket.SSL_ERROR_WANT_READ:
105 continue
105 continue
106 else:
106 else:
107 raise x
107 raise x
108
108
109 _PROTOCOL_SSLv23 = 2
109 _PROTOCOL_SSLv3 = 1
110
110
111 CERT_NONE = 0
111 CERT_NONE = 0
112 CERT_OPTIONAL = 1
112 CERT_OPTIONAL = 1
113 CERT_REQUIRED = 2
113 CERT_REQUIRED = 2
114
114
115 # Disable unused-argument because we're making a dumb wrapper
115 # Disable unused-argument because we're making a dumb wrapper
116 # that's like an upstream method.
116 # that's like an upstream method.
117 #
117 #
118 # pylint: disable=W0613,R0913
118 # pylint: disable=W0613,R0913
119 def wrap_socket(sock, keyfile=None, certfile=None,
119 def wrap_socket(sock, keyfile=None, certfile=None,
120 server_side=False, cert_reqs=CERT_NONE,
120 server_side=False, cert_reqs=CERT_NONE,
121 ssl_version=_PROTOCOL_SSLv23, ca_certs=None,
121 ssl_version=_PROTOCOL_SSLv3, ca_certs=None,
122 do_handshake_on_connect=True,
122 do_handshake_on_connect=True,
123 suppress_ragged_eofs=True):
123 suppress_ragged_eofs=True):
124 """Backport of ssl.wrap_socket from Python 2.6."""
124 """Backport of ssl.wrap_socket from Python 2.6."""
125 if cert_reqs != CERT_NONE and ca_certs:
125 if cert_reqs != CERT_NONE and ca_certs:
126 raise CertificateValidationUnsupported(
126 raise CertificateValidationUnsupported(
127 'SSL certificate validation requires the ssl module'
127 'SSL certificate validation requires the ssl module'
128 '(included in Python 2.6 and later.)')
128 '(included in Python 2.6 and later.)')
129 sslob = socket.ssl(sock)
129 sslob = socket.ssl(sock)
130 # borrow httplib's workaround for no ssl.wrap_socket
130 # borrow httplib's workaround for no ssl.wrap_socket
131 sock = FakeSocket(sock, sslob)
131 sock = FakeSocket(sock, sslob)
132 return sock
132 return sock
133 # pylint: enable=W0613,R0913
133 # pylint: enable=W0613,R0913
134
134
135
135
136 class CertificateValidationUnsupported(Exception):
136 class CertificateValidationUnsupported(Exception):
137 """Exception raised when cert validation is requested but unavailable."""
137 """Exception raised when cert validation is requested but unavailable."""
138 # no-check-code
138 # no-check-code
General Comments 0
You need to be logged in to leave comments. Login now