keepalive.py
709 lines
| 23.0 KiB
| text/x-python
|
PythonLexer
/ mercurial / keepalive.py
|
r2435 | # This library is free software; you can redistribute it and/or | ||
# modify it under the terms of the GNU Lesser General Public | ||||
# License as published by the Free Software Foundation; either | ||||
# version 2.1 of the License, or (at your option) any later version. | ||||
# | ||||
# This library is distributed in the hope that it will be useful, | ||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||||
# Lesser General Public License for more details. | ||||
# | ||||
# You should have received a copy of the GNU Lesser General Public | ||||
|
r15782 | # License along with this library; if not, see | ||
# <http://www.gnu.org/licenses/>. | ||||
|
r2435 | |||
# This file is part of urlgrabber, a high-level cross-protocol url-grabber | ||||
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko | ||||
|
r4026 | # Modified by Benoit Boissinot: | ||
# - fix for digest auth (inspired from urllib2.py @ Python v2.4) | ||||
|
r6470 | # Modified by Dirkjan Ochtman: | ||
# - import md5 function from a local util module | ||||
|
r9726 | # Modified by Augie Fackler: | ||
# - add safesend method and use it to prevent broken pipe errors | ||||
# on large POST requests | ||||
|
r4026 | |||
|
r2435 | """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive. | ||
>>> import urllib2 | ||||
>>> from keepalive import HTTPHandler | ||||
>>> keepalive_handler = HTTPHandler() | ||||
|
r28883 | >>> opener = urlreq.buildopener(keepalive_handler) | ||
>>> urlreq.installopener(opener) | ||||
|
r2600 | >>> | ||
|
r28883 | >>> fo = urlreq.urlopen('http://www.python.org') | ||
|
r2435 | |||
If a connection to a given host is requested, and all of the existing | ||||
connections are still in use, another connection will be opened. If | ||||
the handler tries to use an existing connection but it fails in some | ||||
way, it will be closed and removed from the pool. | ||||
To remove the handler, simply re-run build_opener with no arguments, and | ||||
install that opener. | ||||
You can explicitly close connections by using the close_connection() | ||||
method of the returned file-like object (described below) or you can | ||||
use the handler methods: | ||||
close_connection(host) | ||||
close_all() | ||||
open_connections() | ||||
NOTE: using the close_connection and close_all methods of the handler | ||||
should be done with care when using multiple threads. | ||||
* there is nothing that prevents another thread from creating new | ||||
connections immediately after connections are closed | ||||
* no checks are done to prevent in-use connections from being closed | ||||
>>> keepalive_handler.close_all() | ||||
EXTRA ATTRIBUTES AND METHODS | ||||
Upon a status of 200, the object returned has a few additional | ||||
attributes and methods, which should not be used if you want to | ||||
remain consistent with the normal urllib2-returned objects: | ||||
close_connection() - close the connection to the host | ||||
readlines() - you know, readlines() | ||||
|
r17424 | status - the return status (i.e. 404) | ||
reason - english translation of status (i.e. 'File not found') | ||||
|
r2435 | |||
If you want the best of both worlds, use this inside an | ||||
AttributeError-catching try: | ||||
>>> try: status = fo.status | ||||
>>> except AttributeError: status = None | ||||
Unfortunately, these are ONLY there if status == 200, so it's not | ||||
easy to distinguish between non-200 responses. The reason is that | ||||
urllib2 tries to do clever things with error codes 301, 302, 401, | ||||
and 407, and it wraps the object upon return. | ||||
""" | ||||
|
r2444 | # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $ | ||
|
r2435 | |||
|
r27616 | from __future__ import absolute_import, print_function | ||
|
r27507 | |||
|
r9726 | import errno | ||
|
r29341 | import hashlib | ||
|
r2435 | import socket | ||
|
r27507 | import sys | ||
|
r29456 | import threading | ||
|
r28883 | |||
from . import ( | ||||
util, | ||||
) | ||||
|
r29455 | httplib = util.httplib | ||
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
|
r2435 | |||
DEBUG = None | ||||
|
r14764 | class ConnectionManager(object): | ||
|
r2435 | """ | ||
The connection manager must be able to: | ||||
* keep track of all existing | ||||
""" | ||||
def __init__(self): | ||||
|
r29456 | self._lock = threading.Lock() | ||
|
r2435 | self._hostmap = {} # map hosts to a list of connections | ||
self._connmap = {} # map connections to host | ||||
self._readymap = {} # map connection to ready state | ||||
def add(self, host, connection, ready): | ||||
self._lock.acquire() | ||||
try: | ||||
|
r16686 | if host not in self._hostmap: | ||
|
r10282 | self._hostmap[host] = [] | ||
|
r2435 | self._hostmap[host].append(connection) | ||
self._connmap[connection] = host | ||||
self._readymap[connection] = ready | ||||
finally: | ||||
self._lock.release() | ||||
def remove(self, connection): | ||||
self._lock.acquire() | ||||
try: | ||||
try: | ||||
host = self._connmap[connection] | ||||
except KeyError: | ||||
pass | ||||
else: | ||||
del self._connmap[connection] | ||||
del self._readymap[connection] | ||||
self._hostmap[host].remove(connection) | ||||
if not self._hostmap[host]: del self._hostmap[host] | ||||
finally: | ||||
self._lock.release() | ||||
def set_ready(self, connection, ready): | ||||
|
r10282 | try: | ||
self._readymap[connection] = ready | ||||
except KeyError: | ||||
pass | ||||
|
r2600 | |||
|
r2435 | def get_ready_conn(self, host): | ||
conn = None | ||||
self._lock.acquire() | ||||
try: | ||||
|
r5915 | if host in self._hostmap: | ||
|
r2435 | for c in self._hostmap[host]: | ||
if self._readymap[c]: | ||||
self._readymap[c] = 0 | ||||
conn = c | ||||
break | ||||
finally: | ||||
self._lock.release() | ||||
return conn | ||||
def get_all(self, host=None): | ||||
if host: | ||||
return list(self._hostmap.get(host, [])) | ||||
else: | ||||
return dict(self._hostmap) | ||||
|
r14764 | class KeepAliveHandler(object): | ||
|
r2435 | def __init__(self): | ||
self._cm = ConnectionManager() | ||||
|
r2600 | |||
|
r2435 | #### Connection Management | ||
def open_connections(self): | ||||
"""return a list of connected hosts and the number of connections | ||||
to each. [('foo.com:80', 2), ('bar.org', 1)]""" | ||||
return [(host, len(li)) for (host, li) in self._cm.get_all().items()] | ||||
def close_connection(self, host): | ||||
"""close connection(s) to <host> | ||||
host is the host:port spec, as in 'www.cnn.com:8080' as passed in. | ||||
no error occurs if there is no connection to that host.""" | ||||
for h in self._cm.get_all(host): | ||||
self._cm.remove(h) | ||||
h.close() | ||||
|
r2600 | |||
|
r2435 | def close_all(self): | ||
"""close all open connections""" | ||||
|
r7622 | for host, conns in self._cm.get_all().iteritems(): | ||
|
r2435 | for h in conns: | ||
self._cm.remove(h) | ||||
h.close() | ||||
|
r2600 | |||
|
r2435 | def _request_closed(self, request, host, connection): | ||
|
r17251 | """tells us that this request is now closed and that the | ||
|
r2435 | connection is ready for another request""" | ||
self._cm.set_ready(connection, 1) | ||||
def _remove_connection(self, host, connection, close=0): | ||||
|
r10282 | if close: | ||
connection.close() | ||||
|
r2435 | self._cm.remove(connection) | ||
|
r2600 | |||
|
r2435 | #### Transaction Execution | ||
def http_open(self, req): | ||||
return self.do_open(HTTPConnection, req) | ||||
def do_open(self, http_class, req): | ||||
host = req.get_host() | ||||
if not host: | ||||
|
r28883 | raise urlerr.urlerror('no host given') | ||
|
r2435 | |||
try: | ||||
h = self._cm.get_ready_conn(host) | ||||
while h: | ||||
r = self._reuse_connection(h, req, host) | ||||
# if this response is non-None, then it worked and we're | ||||
# done. Break out, skipping the else block. | ||||
|
r10282 | if r: | ||
break | ||||
|
r2435 | |||
# connection is bad - possibly closed by server | ||||
# discard it and ask for the next free connection | ||||
h.close() | ||||
self._cm.remove(h) | ||||
h = self._cm.get_ready_conn(host) | ||||
else: | ||||
# no (working) free connections were found. Create a new one. | ||||
h = http_class(host) | ||||
|
r10282 | if DEBUG: | ||
DEBUG.info("creating new connection to %s (%d)", | ||||
host, id(h)) | ||||
|
r2435 | self._cm.add(host, h, 0) | ||
self._start_transaction(h, req) | ||||
r = h.getresponse() | ||||
|
r25660 | except (socket.error, httplib.HTTPException) as err: | ||
|
r28883 | raise urlerr.urlerror(err) | ||
|
r2600 | |||
|
r2435 | # if not a persistent connection, don't try to reuse it | ||
|
r10282 | if r.will_close: | ||
self._cm.remove(h) | ||||
|
r2435 | |||
|
r10282 | if DEBUG: | ||
DEBUG.info("STATUS: %s, %s", r.status, r.reason) | ||||
|
r2435 | r._handler = self | ||
r._host = host | ||||
r._url = req.get_full_url() | ||||
r._connection = h | ||||
r.code = r.status | ||||
|
r2444 | r.headers = r.msg | ||
r.msg = r.reason | ||||
|
r2600 | |||
|
r30487 | return r | ||
|
r2435 | |||
def _reuse_connection(self, h, req, host): | ||||
"""start the transaction with a re-used connection | ||||
return a response object (r) upon success or None on failure. | ||||
This DOES not close or remove bad connections in cases where | ||||
it returns. However, if an unexpected exception occurs, it | ||||
will close and remove the connection before re-raising. | ||||
""" | ||||
try: | ||||
self._start_transaction(h, req) | ||||
r = h.getresponse() | ||||
# note: just because we got something back doesn't mean it | ||||
# worked. We'll check the version below, too. | ||||
except (socket.error, httplib.HTTPException): | ||||
r = None | ||||
|
r16705 | except: # re-raises | ||
|
r2435 | # adding this block just in case we've missed | ||
# something we will still raise the exception, but | ||||
# lets try and close the connection and remove it | ||||
# first. We previously got into a nasty loop | ||||
# where an exception was uncaught, and so the | ||||
# connection stayed open. On the next try, the | ||||
|
r17424 | # same exception was raised, etc. The trade-off is | ||
|
r2435 | # that it's now possible this call will raise | ||
# a DIFFERENT exception | ||||
|
r10282 | if DEBUG: | ||
DEBUG.error("unexpected exception - closing " | ||||
"connection to %s (%d)", host, id(h)) | ||||
|
r2435 | self._cm.remove(h) | ||
h.close() | ||||
raise | ||||
|
r2600 | |||
|
r2435 | if r is None or r.version == 9: | ||
# httplib falls back to assuming HTTP 0.9 if it gets a | ||||
# bad header back. This is most likely to happen if | ||||
# the socket has been closed by the server since we | ||||
# last used the connection. | ||||
|
r10282 | if DEBUG: | ||
DEBUG.info("failed to re-use connection to %s (%d)", | ||||
host, id(h)) | ||||
|
r2435 | r = None | ||
else: | ||||
|
r10282 | if DEBUG: | ||
DEBUG.info("re-using connection to %s (%d)", host, id(h)) | ||||
|
r2435 | |||
return r | ||||
def _start_transaction(self, h, req): | ||||
|
r8233 | # What follows mostly reimplements HTTPConnection.request() | ||
|
r31999 | # except it adds self.parent.addheaders in the mix and sends headers | ||
# in a deterministic order (to make testing easier). | ||||
headers = util.sortdict(self.parent.addheaders) | ||||
headers.update(sorted(req.headers.items())) | ||||
headers.update(sorted(req.unredirected_hdrs.items())) | ||||
headers = util.sortdict((n.lower(), v) for n, v in headers.items()) | ||||
|
r8233 | skipheaders = {} | ||
for n in ('host', 'accept-encoding'): | ||||
if n in headers: | ||||
skipheaders['skip_' + n.replace('-', '_')] = 1 | ||||
|
r2435 | try: | ||
|
r8146 | if req.has_data(): | ||
data = req.get_data() | ||||
|
r30922 | h.putrequest( | ||
req.get_method(), req.get_selector(), **skipheaders) | ||||
|
r8233 | if 'content-type' not in headers: | ||
|
r8146 | h.putheader('Content-type', | ||
'application/x-www-form-urlencoded') | ||||
|
r8233 | if 'content-length' not in headers: | ||
|
r8146 | h.putheader('Content-length', '%d' % len(data)) | ||
else: | ||||
|
r30922 | h.putrequest( | ||
req.get_method(), req.get_selector(), **skipheaders) | ||||
|
r28278 | except socket.error as err: | ||
|
r28883 | raise urlerr.urlerror(err) | ||
|
r8233 | for k, v in headers.items(): | ||
|
r8146 | h.putheader(k, v) | ||
h.endheaders() | ||||
if req.has_data(): | ||||
h.send(data) | ||||
|
r28883 | class HTTPHandler(KeepAliveHandler, urlreq.httphandler): | ||
|
r5983 | pass | ||
|
r2435 | class HTTPResponse(httplib.HTTPResponse): | ||
# we need to subclass HTTPResponse in order to | ||||
# 1) add readline() and readlines() methods | ||||
# 2) add close_connection() methods | ||||
# 3) add info() and geturl() methods | ||||
# in order to add readline(), read must be modified to deal with a | ||||
# buffer. example: readline must read a buffer and then spit back | ||||
# one line at a time. The only real alternative is to read one | ||||
# BYTE at a time (ick). Once something has been read, it can't be | ||||
# put back (ok, maybe it can, but that's even uglier than this), | ||||
# so if you THEN do a normal read, you must first take stuff from | ||||
# the buffer. | ||||
|
r17424 | # the read method wraps the original to accommodate buffering, | ||
|
r2435 | # although read() never adds to the buffer. | ||
# Both readline and readlines have been stolen with almost no | ||||
# modification from socket.py | ||||
|
r2600 | |||
|
r2435 | |||
def __init__(self, sock, debuglevel=0, strict=0, method=None): | ||||
|
r17700 | httplib.HTTPResponse.__init__(self, sock, debuglevel, method) | ||
|
r2435 | self.fileno = sock.fileno | ||
self.code = None | ||||
self._rbuf = '' | ||||
self._rbufsize = 8096 | ||||
self._handler = None # inserted by the handler later | ||||
self._host = None # (same) | ||||
self._url = None # (same) | ||||
self._connection = None # (same) | ||||
_raw_read = httplib.HTTPResponse.read | ||||
def close(self): | ||||
if self.fp: | ||||
self.fp.close() | ||||
self.fp = None | ||||
if self._handler: | ||||
self._handler._request_closed(self, self._host, | ||||
self._connection) | ||||
def close_connection(self): | ||||
self._handler._remove_connection(self._host, self._connection, close=1) | ||||
self.close() | ||||
|
r2600 | |||
|
r2435 | def info(self): | ||
|
r2444 | return self.headers | ||
|
r2435 | |||
def geturl(self): | ||||
return self._url | ||||
def read(self, amt=None): | ||||
# the _rbuf test is only in this first if for speed. It's not | ||||
# logically necessary | ||||
if self._rbuf and not amt is None: | ||||
L = len(self._rbuf) | ||||
if amt > L: | ||||
amt -= L | ||||
else: | ||||
s = self._rbuf[:amt] | ||||
self._rbuf = self._rbuf[amt:] | ||||
return s | ||||
s = self._rbuf + self._raw_read(amt) | ||||
self._rbuf = '' | ||||
return s | ||||
|
r7781 | # stolen from Python SVN #68532 to fix issue1088 | ||
def _read_chunked(self, amt): | ||||
chunk_left = self.chunk_left | ||||
|
r30686 | parts = [] | ||
|
r7781 | |||
while True: | ||||
if chunk_left is None: | ||||
line = self.fp.readline() | ||||
i = line.find(';') | ||||
if i >= 0: | ||||
line = line[:i] # strip chunk-extensions | ||||
try: | ||||
chunk_left = int(line, 16) | ||||
except ValueError: | ||||
|
r17424 | # close the connection as protocol synchronization is | ||
|
r7781 | # probably lost | ||
self.close() | ||||
|
r30686 | raise httplib.IncompleteRead(''.join(parts)) | ||
|
r7781 | if chunk_left == 0: | ||
break | ||||
if amt is None: | ||||
|
r30686 | parts.append(self._safe_read(chunk_left)) | ||
|
r7781 | elif amt < chunk_left: | ||
|
r30686 | parts.append(self._safe_read(amt)) | ||
|
r7781 | self.chunk_left = chunk_left - amt | ||
|
r30686 | return ''.join(parts) | ||
|
r7781 | elif amt == chunk_left: | ||
|
r30686 | parts.append(self._safe_read(amt)) | ||
|
r7781 | self._safe_read(2) # toss the CRLF at the end of the chunk | ||
self.chunk_left = None | ||||
|
r30686 | return ''.join(parts) | ||
|
r7781 | else: | ||
|
r30686 | parts.append(self._safe_read(chunk_left)) | ||
|
r7781 | amt -= chunk_left | ||
# we read the whole chunk, get another | ||||
self._safe_read(2) # toss the CRLF at the end of the chunk | ||||
chunk_left = None | ||||
# read and discard trailer up to the CRLF terminator | ||||
### note: we shouldn't have any trailers! | ||||
while True: | ||||
line = self.fp.readline() | ||||
if not line: | ||||
# a vanishingly small number of sites EOF without | ||||
# sending the trailer | ||||
break | ||||
if line == '\r\n': | ||||
break | ||||
# we read everything; close the "file" | ||||
self.close() | ||||
|
r30686 | return ''.join(parts) | ||
|
r7781 | |||
|
r30687 | def readline(self): | ||
|
r30688 | # Fast path for a line is already available in read buffer. | ||
|
r2435 | i = self._rbuf.find('\n') | ||
|
r30688 | if i >= 0: | ||
i += 1 | ||||
line = self._rbuf[:i] | ||||
self._rbuf = self._rbuf[i:] | ||||
return line | ||||
# No newline in local buffer. Read until we find one. | ||||
chunks = [self._rbuf] | ||||
i = -1 | ||||
readsize = self._rbufsize | ||||
while True: | ||||
new = self._raw_read(readsize) | ||||
|
r10282 | if not new: | ||
break | ||||
|
r30688 | |||
chunks.append(new) | ||||
|
r2435 | i = new.find('\n') | ||
|
r10282 | if i >= 0: | ||
|
r30688 | break | ||
# We either have exhausted the stream or have a newline in chunks[-1]. | ||||
# EOF | ||||
if i == -1: | ||||
self._rbuf = '' | ||||
return ''.join(chunks) | ||||
i += 1 | ||||
self._rbuf = chunks[-1][i:] | ||||
chunks[-1] = chunks[-1][:i] | ||||
return ''.join(chunks) | ||||
|
r2435 | |||
|
r19872 | def readlines(self, sizehint=0): | ||
|
r2435 | total = 0 | ||
list = [] | ||||
|
r14494 | while True: | ||
|
r2435 | line = self.readline() | ||
|
r10282 | if not line: | ||
break | ||||
|
r2435 | list.append(line) | ||
total += len(line) | ||||
if sizehint and total >= sizehint: | ||||
break | ||||
return list | ||||
|
r9726 | def safesend(self, str): | ||
"""Send `str' to the server. | ||||
Shamelessly ripped off from httplib to patch a bad behavior. | ||||
""" | ||||
# _broken_pipe_resp is an attribute we set in this function | ||||
# if the socket is closed while we're sending data but | ||||
# the server sent us a response before hanging up. | ||||
# In that case, we want to pretend to send the rest of the | ||||
# outgoing data, and then let the user use getresponse() | ||||
# (which we wrap) to get this last response before | ||||
# opening a new socket. | ||||
if getattr(self, '_broken_pipe_resp', None) is not None: | ||||
return | ||||
if self.sock is None: | ||||
if self.auto_open: | ||||
self.connect() | ||||
else: | ||||
|
r16687 | raise httplib.NotConnected | ||
|
r9726 | |||
# send the data to the server. if we get a broken pipe, then close | ||||
# the socket. we want to reconnect when somebody tries to send again. | ||||
# | ||||
# NOTE: we DO propagate the error, though, because we cannot simply | ||||
# ignore the error... the caller will know if they can retry. | ||||
if self.debuglevel > 0: | ||||
|
r27616 | print("send:", repr(str)) | ||
|
r9726 | try: | ||
|
r10282 | blocksize = 8192 | ||
|
r14958 | read = getattr(str, 'read', None) | ||
if read is not None: | ||||
|
r10282 | if self.debuglevel > 0: | ||
|
r27616 | print("sending a read()able") | ||
|
r14958 | data = read(blocksize) | ||
|
r9726 | while data: | ||
self.sock.sendall(data) | ||||
|
r14958 | data = read(blocksize) | ||
|
r9726 | else: | ||
self.sock.sendall(str) | ||||
|
r25660 | except socket.error as v: | ||
|
r9726 | reraise = True | ||
if v[0] == errno.EPIPE: # Broken pipe | ||||
if self._HTTPConnection__state == httplib._CS_REQ_SENT: | ||||
self._broken_pipe_resp = None | ||||
self._broken_pipe_resp = self.getresponse() | ||||
reraise = False | ||||
self.close() | ||||
if reraise: | ||||
raise | ||||
def wrapgetresponse(cls): | ||||
"""Wraps getresponse in cls with a broken-pipe sane version. | ||||
""" | ||||
def safegetresponse(self): | ||||
# In safesend() we might set the _broken_pipe_resp | ||||
# attribute, in which case the socket has already | ||||
# been closed and we just need to give them the response | ||||
# back. Otherwise, we use the normal response path. | ||||
r = getattr(self, '_broken_pipe_resp', None) | ||||
if r is not None: | ||||
return r | ||||
return cls.getresponse(self) | ||||
safegetresponse.__doc__ = cls.getresponse.__doc__ | ||||
return safegetresponse | ||||
|
r2435 | |||
class HTTPConnection(httplib.HTTPConnection): | ||||
# use the modified response class | ||||
response_class = HTTPResponse | ||||
|
r9726 | send = safesend | ||
getresponse = wrapgetresponse(httplib.HTTPConnection) | ||||
|
r2600 | |||
|
r2435 | ######################################################################### | ||
##### TEST FUNCTIONS | ||||
######################################################################### | ||||
def continuity(url): | ||||
|
r29341 | md5 = hashlib.md5 | ||
|
r2435 | format = '%25s: %s' | ||
|
r2600 | |||
|
r2435 | # first fetch the file with the normal http handler | ||
|
r28883 | opener = urlreq.buildopener() | ||
urlreq.installopener(opener) | ||||
fo = urlreq.urlopen(url) | ||||
|
r2435 | foo = fo.read() | ||
fo.close() | ||||
|
r22505 | m = md5(foo) | ||
|
r27616 | print(format % ('normal urllib', m.hexdigest())) | ||
|
r2435 | |||
# now install the keepalive handler and try again | ||||
|
r28883 | opener = urlreq.buildopener(HTTPHandler()) | ||
urlreq.installopener(opener) | ||||
|
r2435 | |||
|
r28883 | fo = urlreq.urlopen(url) | ||
|
r2435 | foo = fo.read() | ||
fo.close() | ||||
|
r22505 | m = md5(foo) | ||
|
r27616 | print(format % ('keepalive read', m.hexdigest())) | ||
|
r2435 | |||
|
r28883 | fo = urlreq.urlopen(url) | ||
|
r2435 | foo = '' | ||
|
r14494 | while True: | ||
|
r2435 | f = fo.readline() | ||
|
r10282 | if f: | ||
foo = foo + f | ||||
|
r2435 | else: break | ||
fo.close() | ||||
|
r22505 | m = md5(foo) | ||
|
r27616 | print(format % ('keepalive readline', m.hexdigest())) | ||
|
r2435 | |||
def comp(N, url): | ||||
|
r27616 | print(' making %i connections to:\n %s' % (N, url)) | ||
|
r2435 | |||
|
r30473 | util.stdout.write(' first using the normal urllib handlers') | ||
|
r2435 | # first use normal opener | ||
|
r28883 | opener = urlreq.buildopener() | ||
urlreq.installopener(opener) | ||||
|
r2435 | t1 = fetch(N, url) | ||
|
r27616 | print(' TIME: %.3f s' % t1) | ||
|
r2435 | |||
|
r30473 | util.stdout.write(' now using the keepalive handler ') | ||
|
r2435 | # now install the keepalive handler and try again | ||
|
r28883 | opener = urlreq.buildopener(HTTPHandler()) | ||
urlreq.installopener(opener) | ||||
|
r2435 | t2 = fetch(N, url) | ||
|
r27616 | print(' TIME: %.3f s' % t2) | ||
print(' improvement factor: %.2f' % (t1 / t2)) | ||||
|
r2600 | |||
|
r2435 | def fetch(N, url, delay=0): | ||
import time | ||||
lens = [] | ||||
starttime = time.time() | ||||
for i in range(N): | ||||
|
r10282 | if delay and i > 0: | ||
time.sleep(delay) | ||||
|
r28883 | fo = urlreq.urlopen(url) | ||
|
r2435 | foo = fo.read() | ||
fo.close() | ||||
lens.append(len(foo)) | ||||
diff = time.time() - starttime | ||||
j = 0 | ||||
for i in lens[1:]: | ||||
j = j + 1 | ||||
if not i == lens[0]: | ||||
|
r27616 | print("WARNING: inconsistent length on read %i: %i" % (j, i)) | ||
|
r2435 | |||
return diff | ||||
def test_timeout(url): | ||||
global DEBUG | ||||
dbbackup = DEBUG | ||||
|
r14764 | class FakeLogger(object): | ||
|
r10282 | def debug(self, msg, *args): | ||
|
r27616 | print(msg % args) | ||
|
r2435 | info = warning = error = debug | ||
DEBUG = FakeLogger() | ||||
|
r27616 | print(" fetching the file to establish a connection") | ||
|
r28883 | fo = urlreq.urlopen(url) | ||
|
r2435 | data1 = fo.read() | ||
fo.close() | ||||
|
r2600 | |||
|
r2435 | i = 20 | ||
|
r27616 | print(" waiting %i seconds for the server to close the connection" % i) | ||
|
r2435 | while i > 0: | ||
|
r30473 | util.stdout.write('\r %2i' % i) | ||
util.stdout.flush() | ||||
|
r2435 | time.sleep(1) | ||
i -= 1 | ||||
|
r30473 | util.stderr.write('\r') | ||
|
r2435 | |||
|
r27616 | print(" fetching the file a second time") | ||
|
r28883 | fo = urlreq.urlopen(url) | ||
|
r2435 | data2 = fo.read() | ||
fo.close() | ||||
if data1 == data2: | ||||
|
r27616 | print(' data are identical') | ||
|
r2435 | else: | ||
|
r27616 | print(' ERROR: DATA DIFFER') | ||
|
r2435 | |||
DEBUG = dbbackup | ||||
|
r2600 | |||
|
r2435 | def test(url, N=10): | ||
|
r27616 | print("performing continuity test (making sure stuff isn't corrupted)") | ||
|
r2435 | continuity(url) | ||
|
r27616 | print('') | ||
print("performing speed comparison") | ||||
|
r2435 | comp(N, url) | ||
|
r27616 | print('') | ||
print("performing dropped-connection check") | ||||
|
r2435 | test_timeout(url) | ||
|
r2600 | |||
|
r2435 | if __name__ == '__main__': | ||
import time | ||||
try: | ||||
N = int(sys.argv[1]) | ||||
url = sys.argv[2] | ||||
|
r16688 | except (IndexError, ValueError): | ||
|
r27616 | print("%s <integer> <url>" % sys.argv[0]) | ||
|
r2435 | else: | ||
test(url, N) | ||||