Import new http library as mercurial.httpclient.

This is revision a4229f13c374 of
http://py-nonblocking-http.googlecode.com/ with a no-check-code
comment added to the end of each file using `for fi in $(hg manifest |
grep mercurial/httpclient/) ; echo '# no-check-code' >> $fi`.

mercurial/httpclient/__init__.py

@@ -0,0 +1,650 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Improved HTTP/1.1 client library
+
+This library contains an HTTPConnection which is similar to the one in
+httplib, but has several additional features:
+
+  * supports keepalives natively
+  * uses select() to block for incoming data
+  * notices when the server responds early to a request
+  * implements ssl inline instead of in a different class
+"""
+
+import cStringIO
+import errno
+import httplib
+import logging
+import rfc822
+import select
+import socket
+
+import socketutil
+
+logger = logging.getLogger(__name__)
+
+__all__ = ['HTTPConnection', 'HTTPResponse']
+
+HTTP_VER_1_0 = 'HTTP/1.0'
+HTTP_VER_1_1 = 'HTTP/1.1'
+
+_LEN_CLOSE_IS_END = -1
+
+OUTGOING_BUFFER_SIZE = 1 << 15
+INCOMING_BUFFER_SIZE = 1 << 20
+
+HDR_ACCEPT_ENCODING = 'accept-encoding'
+HDR_CONNECTION_CTRL = 'connection'
+HDR_CONTENT_LENGTH = 'content-length'
+HDR_XFER_ENCODING = 'transfer-encoding'
+
+XFER_ENCODING_CHUNKED = 'chunked'
+
+CONNECTION_CLOSE = 'close'
+
+EOL = '\r\n'
+_END_HEADERS = EOL * 2
+
+# Based on some searching around, 1 second seems like a reasonable
+# default here.
+TIMEOUT_ASSUME_CONTINUE = 1
+TIMEOUT_DEFAULT = None
+
+
+class HTTPResponse(object):
+    """Response from an HTTP server.
+
+    The response will continue to load as available. If you need the
+    complete response before continuing, check the .complete() method.
+    """
+    def __init__(self, sock, timeout):
+        self.sock = sock
+        self.raw_response = ''
+        self._body = None
+        self._headers_len = 0
+        self._content_len = 0
+        self.headers = None
+        self.will_close = False
+        self.status_line = ''
+        self.status = None
+        self.http_version = None
+        self.reason = None
+        self._chunked = False
+        self._chunked_done = False
+        self._chunked_until_next = 0
+        self._chunked_skip_bytes = 0
+        self._chunked_preloaded_block = None
+
+        self._read_location = 0
+        self._eol = EOL
+
+        self._timeout = timeout
+
+    @property
+    def _end_headers(self):
+        return self._eol * 2
+
+    def complete(self):
+        """Returns true if this response is completely loaded.
+        """
+        if self._chunked:
+            return self._chunked_done
+        if self._content_len == _LEN_CLOSE_IS_END:
+            return False
+        return self._body is not None and len(self._body) >= self._content_len
+
+    def readline(self):
+        """Read a single line from the response body.
+
+        This may block until either a line ending is found or the
+        response is complete.
+        """
+        eol = self._body.find('\n', self._read_location)
+        while eol == -1 and not self.complete():
+            self._select()
+            eol = self._body.find('\n', self._read_location)
+        if eol != -1:
+            eol += 1
+        else:
+            eol = len(self._body)
+        data = self._body[self._read_location:eol]
+        self._read_location = eol
+        return data
+
+    def read(self, length=None):
+        # if length is None, unbounded read
+        while (not self.complete()  # never select on a finished read
+               and (not length  # unbounded, so we wait for complete()
+                    or (self._read_location + length) > len(self._body))):
+            self._select()
+        if not length:
+            length = len(self._body) - self._read_location
+        elif len(self._body) < (self._read_location + length):
+            length = len(self._body) - self._read_location
+        r = self._body[self._read_location:self._read_location + length]
+        self._read_location += len(r)
+        if self.complete() and self.will_close:
+            self.sock.close()
+        return r
+
+    def _select(self):
+        r, _, _ = select.select([self.sock], [], [], self._timeout)
+        if not r:
+            # socket was not readable. If the response is not complete
+            # and we're not a _LEN_CLOSE_IS_END response, raise a timeout.
+            # If we are a _LEN_CLOSE_IS_END response and we have no data,
+            # raise a timeout.
+            if not (self.complete() or
+                    (self._content_len == _LEN_CLOSE_IS_END and self._body)):
+                logger.info('timed out with timeout of %s', self._timeout)
+                raise HTTPTimeoutException('timeout reading data')
+            logger.info('cl: %r body: %r', self._content_len, self._body)
+        data = self.sock.recv(INCOMING_BUFFER_SIZE)
+        logger.debug('response read %d data during _select', len(data))
+        if not data:
+            if not self.headers:
+                self._load_response(self._end_headers)
+                self._content_len = 0
+            elif self._content_len == _LEN_CLOSE_IS_END:
+                self._content_len = len(self._body)
+            return False
+        else:
+            self._load_response(data)
+            return True
+
+    def _chunked_parsedata(self, data):
+        if self._chunked_preloaded_block:
+            data = self._chunked_preloaded_block + data
+            self._chunked_preloaded_block = None
+        while data:
+            logger.debug('looping with %d data remaining', len(data))
+            # Slice out anything we should skip
+            if self._chunked_skip_bytes:
+                if len(data) <= self._chunked_skip_bytes:
+                    self._chunked_skip_bytes -= len(data)
+                    data = ''
+                    break
+                else:
+                    data = data[self._chunked_skip_bytes:]
+                    self._chunked_skip_bytes = 0
+
+            # determine how much is until the next chunk
+            if self._chunked_until_next:
+                amt = self._chunked_until_next
+                logger.debug('reading remaining %d of existing chunk', amt)
+                self._chunked_until_next = 0
+                body = data
+            else:
+                try:
+                    amt, body = data.split(self._eol, 1)
+                except ValueError:
+                    self._chunked_preloaded_block = data
+                    logger.debug('saving %r as a preloaded block for chunked',
+                                 self._chunked_preloaded_block)
+                    return
+                amt = int(amt, base=16)
+                logger.debug('reading chunk of length %d', amt)
+                if amt == 0:
+                    self._chunked_done = True
+
+            # read through end of what we have or the chunk
+            self._body += body[:amt]
+            if len(body) >= amt:
+                data = body[amt:]
+                self._chunked_skip_bytes = len(self._eol)
+            else:
+                self._chunked_until_next = amt - len(body)
+                self._chunked_skip_bytes = 0
+                data = ''
+
+    def _load_response(self, data):
+        if self._chunked:
+            self._chunked_parsedata(data)
+            return
+        elif self._body is not None:
+            self._body += data
+            return
+
+        # We haven't seen end of headers yet
+        self.raw_response += data
+        # This is a bogus server with bad line endings
+        if self._eol not in self.raw_response:
+            for bad_eol in ('\n', '\r'):
+                if (bad_eol in self.raw_response
+                    # verify that bad_eol is not the end of the incoming data
+                    # as this could be a response line that just got
+                    # split between \r and \n.
+                    and (self.raw_response.index(bad_eol) <
+                         (len(self.raw_response) - 1))):
+                    logger.info('bogus line endings detected, '
+                                'using %r for EOL', bad_eol)
+                    self._eol = bad_eol
+                    break
+        # exit early if not at end of headers
+        if self._end_headers not in self.raw_response or self.headers:
+            return
+
+        # handle 100-continue response
+        hdrs, body = self.raw_response.split(self._end_headers, 1)
+        http_ver, status = hdrs.split(' ', 1)
+        if status.startswith('100'):
+            self.raw_response = body
+            logger.debug('continue seen, setting body to %r', body)
+            return
+
+        # arriving here means we should parse response headers
+        # as all headers have arrived completely
+        hdrs, body = self.raw_response.split(self._end_headers, 1)
+        del self.raw_response
+        if self._eol in hdrs:
+            self.status_line, hdrs = hdrs.split(self._eol, 1)
+        else:
+            self.status_line = hdrs
+            hdrs = ''
+        # TODO HTTP < 1.0 support
+        (self.http_version, self.status,
+         self.reason) = self.status_line.split(' ', 2)
+        self.status = int(self.status)
+        if self._eol != EOL:
+            hdrs = hdrs.replace(self._eol, '\r\n')
+        headers = rfc822.Message(cStringIO.StringIO(hdrs))
+        if HDR_CONTENT_LENGTH in headers:
+            self._content_len = int(headers[HDR_CONTENT_LENGTH])
+        if self.http_version == HTTP_VER_1_0:
+            self.will_close = True
+        elif HDR_CONNECTION_CTRL in headers:
+            self.will_close = (
+                headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
+            if self._content_len == 0:
+                self._content_len = _LEN_CLOSE_IS_END
+        if (HDR_XFER_ENCODING in headers
+            and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
+            self._body = ''
+            self._chunked_parsedata(body)
+            self._chunked = True
+        if self._body is None:
+            self._body = body
+        self.headers = headers
+
+
+class HTTPConnection(object):
+    """Connection to a single http server.
+
+    Supports 100-continue and keepalives natively. Uses select() for
+    non-blocking socket operations.
+    """
+    http_version = HTTP_VER_1_1
+    response_class = HTTPResponse
+
+    def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
+                 timeout=TIMEOUT_DEFAULT,
+                 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
+                 proxy_hostport=None, **ssl_opts):
+        """Create a new HTTPConnection.
+
+        Args:
+          host: The host to which we'll connect.
+          port: Optional. The port over which we'll connect. Default 80 for
+                non-ssl, 443 for ssl.
+          use_ssl: Optional. Wether to use ssl. Defaults to False if port is
+                   not 443, true if port is 443.
+          ssl_validator: a function(socket) to validate the ssl cert
+          timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
+          continue_timeout: Optional. Timeout for waiting on an expected
+                   "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
+          proxy_hostport: Optional. Tuple of (host, port) to use as an http
+                       proxy for the connection. Default is to not use a proxy.
+        """
+        if port is None and host.count(':') == 1 or ']:' in host:
+            host, port = host.rsplit(':', 1)
+            port = int(port)
+            if '[' in host:
+                host = host[1:-1]
+        if use_ssl is None and port is None:
+            use_ssl = False
+            port = 80
+        elif use_ssl is None:
+            use_ssl = (port == 443)
+        elif port is None:
+            port = (use_ssl and 443 or 80)
+        self.port = port
+        if use_ssl and not socketutil.have_ssl:
+            raise Exception('ssl requested but unavailable on this Python')
+        self.ssl = use_ssl
+        self.ssl_opts = ssl_opts
+        self._ssl_validator = ssl_validator
+        self.host = host
+        self.sock = None
+        self._current_response = None
+        self._current_response_taken = False
+        if proxy_hostport is None:
+            self._proxy_host = self._proxy_port = None
+        else:
+            self._proxy_host, self._proxy_port = proxy_hostport
+
+        self.timeout = timeout
+        self.continue_timeout = continue_timeout
+
+    def _connect(self):
+        """Connect to the host and port specified in __init__."""
+        if self.sock:
+            return
+        if self._proxy_host is not None:
+            logger.info('Connecting to http proxy %s:%s',
+                        self._proxy_host, self._proxy_port)
+            sock = socketutil.create_connection((self._proxy_host,
+                                                 self._proxy_port))
+            if self.ssl:
+                # TODO proxy header support
+                data = self.buildheaders('CONNECT', '%s:%d' % (self.host,
+                                                               self.port),
+                                         {}, HTTP_VER_1_0)
+                sock.send(data)
+                sock.setblocking(0)
+                r = self.response_class(sock, self.timeout)
+                timeout_exc = HTTPTimeoutException(
+                    'Timed out waiting for CONNECT response from proxy')
+                while not r.complete():
+                    try:
+                        if not r._select():
+                            raise timeout_exc
+                    except HTTPTimeoutException:
+                        # This raise/except pattern looks goofy, but
+                        # _select can raise the timeout as well as the
+                        # loop body. I wish it wasn't this convoluted,
+                        # but I don't have a better solution
+                        # immediately handy.
+                        raise timeout_exc
+                if r.status != 200:
+                    raise HTTPProxyConnectFailedException(
+                        'Proxy connection failed: %d %s' % (r.status,
+                                                            r.read()))
+                logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
+                            self.host, self.port)
+        else:
+            sock = socketutil.create_connection((self.host, self.port))
+        if self.ssl:
+            logger.debug('wrapping socket for ssl with options %r',
+                         self.ssl_opts)
+            sock = socketutil.wrap_socket(sock, **self.ssl_opts)
+            if self._ssl_validator:
+                self._ssl_validator(sock)
+        sock.setblocking(0)
+        self.sock = sock
+
+    def buildheaders(self, method, path, headers, http_ver):
+        if self.ssl and self.port == 443 or self.port == 80:
+            # default port for protocol, so leave it out
+            hdrhost = self.host
+        else:
+            # include nonstandard port in header
+            if ':' in self.host:  # must be IPv6
+                hdrhost = '[%s]:%d' % (self.host, self.port)
+            else:
+                hdrhost = '%s:%d' % (self.host, self.port)
+        if self._proxy_host and not self.ssl:
+            # When talking to a regular http proxy we must send the
+            # full URI, but in all other cases we must not (although
+            # technically RFC 2616 says servers must accept our
+            # request if we screw up, experimentally few do that
+            # correctly.)
+            assert path[0] == '/', 'path must start with a /'
+            path = 'http://%s%s' % (hdrhost, path)
+        outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
+        headers['host'] = ('Host', hdrhost)
+        headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
+        for hdr, val in headers.itervalues():
+            outgoing.append('%s: %s%s' % (hdr, val, EOL))
+        outgoing.append(EOL)
+        return ''.join(outgoing)
+
+    def close(self):
+        """Close the connection to the server.
+
+        This is a no-op if the connection is already closed. The
+        connection may automatically close if requessted by the server
+        or required by the nature of a response.
+        """
+        if self.sock is None:
+            return
+        self.sock.close()
+        self.sock = None
+        logger.info('closed connection to %s on %s', self.host, self.port)
+
+    def busy(self):
+        """Returns True if this connection object is currently in use.
+
+        If a response is still pending, this will return True, even if
+        the request has finished sending. In the future,
+        HTTPConnection may transparently juggle multiple connections
+        to the server, in which case this will be useful to detect if
+        any of those connections is ready for use.
+        """
+        cr = self._current_response
+        if cr is not None:
+            if self._current_response_taken:
+                if cr.will_close:
+                    self.sock = None
+                    self._current_response = None
+                    return False
+                elif cr.complete():
+                    self._current_response = None
+                    return False
+            return True
+        return False
+
+    def request(self, method, path, body=None, headers={},
+                expect_continue=False):
+        """Send a request to the server.
+
+        For increased flexibility, this does not return the response
+        object. Future versions of HTTPConnection that juggle multiple
+        sockets will be able to send (for example) 5 requests all at
+        once, and then let the requests arrive as data is
+        available. Use the `getresponse()` method to retrieve the
+        response.
+        """
+        if self.busy():
+            raise httplib.CannotSendRequest(
+                'Can not send another request before '
+                'current response is read!')
+        self._current_response_taken = False
+
+        logger.info('sending %s request for %s to %s on port %s',
+                    method, path, self.host, self.port)
+        hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
+        if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
+            expect_continue = True
+        elif expect_continue:
+            hdrs['expect'] = ('Expect', '100-Continue')
+
+        chunked = False
+        if body and HDR_CONTENT_LENGTH not in hdrs:
+            if getattr(body, '__len__', False):
+                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
+            elif getattr(body, 'read', False):
+                hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
+                                           XFER_ENCODING_CHUNKED)
+                chunked = True
+            else:
+                raise BadRequestData('body has no __len__() nor read()')
+
+        self._connect()
+        outgoing_headers = self.buildheaders(
+            method, path, hdrs, self.http_version)
+        response = None
+        first = True
+
+        def reconnect(where):
+            logger.info('reconnecting during %s', where)
+            self.close()
+            self._connect()
+
+        while ((outgoing_headers or body)
+               and not (response and response.complete())):
+            select_timeout = self.timeout
+            out = outgoing_headers or body
+            blocking_on_continue = False
+            if expect_continue and not outgoing_headers and not (
+                response and response.headers):
+                logger.info(
+                    'waiting up to %s seconds for'
+                    ' continue response from server',
+                    self.continue_timeout)
+                select_timeout = self.continue_timeout
+                blocking_on_continue = True
+                out = False
+            if out:
+                w = [self.sock]
+            else:
+                w = []
+            r, w, x = select.select([self.sock], w, [], select_timeout)
+            # if we were expecting a 100 continue and it's been long
+            # enough, just go ahead and assume it's ok. This is the
+            # recommended behavior from the RFC.
+            if r == w == x == []:
+                if blocking_on_continue:
+                    expect_continue = False
+                    logger.info('no response to continue expectation from '
+                                'server, optimistically sending request body')
+                else:
+                    raise HTTPTimeoutException('timeout sending data')
+            # TODO exceptional conditions with select? (what are those be?)
+            # TODO if the response is loading, must we finish sending at all?
+            #
+            # Certainly not if it's going to close the connection and/or
+            # the response is already done...I think.
+            was_first = first
+
+            # incoming data
+            if r:
+                try:
+                    data = r[0].recv(INCOMING_BUFFER_SIZE)
+                    if not data:
+                        logger.info('socket appears closed in read')
+                        outgoing_headers = body = None
+                        break
+                    if response is None:
+                        response = self.response_class(r[0], self.timeout)
+                    response._load_response(data)
+                    if (response._content_len == _LEN_CLOSE_IS_END
+                        and len(data) == 0):
+                        response._content_len = len(response._body)
+                    if response.complete():
+                        w = []
+                        response.will_close = True
+                except socket.error, e:
+                    if e[0] != errno.EPIPE and not was_first:
+                        raise
+                    if (response._content_len
+                        and response._content_len != _LEN_CLOSE_IS_END):
+                        outgoing_headers = sent_data + outgoing_headers
+                        reconnect('read')
+
+            # outgoing data
+            if w and out:
+                try:
+                    if getattr(out, 'read', False):
+                        data = out.read(OUTGOING_BUFFER_SIZE)
+                        if not data:
+                            continue
+                        if len(data) < OUTGOING_BUFFER_SIZE:
+                            if chunked:
+                                body = '0' + EOL + EOL
+                            else:
+                                body = None
+                        if chunked:
+                            out = hex(len(data))[2:] + EOL + data + EOL
+                        else:
+                            out = data
+                    amt = w[0].send(out)
+                except socket.error, e:
+                    if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
+                        # This means that SSL hasn't flushed its buffer into
+                        # the socket yet.
+                        # TODO: find a way to block on ssl flushing its buffer
+                        # similar to selecting on a raw socket.
+                        continue
+                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
+                          and not first):
+                        raise
+                    reconnect('write')
+                    amt = self.sock.send(out)
+                logger.debug('sent %d', amt)
+                first = False
+                # stash data we think we sent in case the socket breaks
+                # when we read from it
+                if was_first:
+                    sent_data = out[:amt]
+                if out is body:
+                    body = out[amt:]
+                else:
+                    outgoing_headers = out[amt:]
+
+        # close if the server response said to or responded before eating
+        # the whole request
+        if response is None:
+            response = self.response_class(self.sock, self.timeout)
+        complete = response.complete()
+        data_left = bool(outgoing_headers or body)
+        if data_left:
+            logger.info('stopped sending request early, '
+                         'will close the socket to be safe.')
+            response.will_close = True
+        if response.will_close:
+            # The socket will be closed by the response, so we disown
+            # the socket
+            self.sock = None
+        self._current_response = response
+
+    def getresponse(self):
+        if self._current_response is None:
+            raise httplib.ResponseNotReady()
+        r = self._current_response
+        while r.headers is None:
+            r._select()
+        if r.complete() or r.will_close:
+            self.sock = None
+            self._current_response = None
+        else:
+            self._current_response_taken = True
+        return r
+
+
+class HTTPTimeoutException(httplib.HTTPException):
+    """A timeout occurred while waiting on the server."""
+
+
+class BadRequestData(httplib.HTTPException):
+    """Request body object has neither __len__ nor read."""
+
+
+class HTTPProxyConnectFailedException(httplib.HTTPException):
+    """Connecting to the HTTP proxy failed."""
+# no-check-code

mercurial/httpclient/socketutil.py

@@ -0,0 +1,134 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Abstraction to simplify socket use for Python < 2.6
+
+This will attempt to use the ssl module and the new
+socket.create_connection method, but fall back to the old
+methods if those are unavailable.
+"""
+import logging
+import socket
+
+logger = logging.getLogger(__name__)
+
+try:
+    import ssl
+    ssl.wrap_socket  # make demandimporters load the module
+    have_ssl = True
+except ImportError:
+    import httplib
+    import urllib2
+    have_ssl = getattr(urllib2, 'HTTPSHandler', False)
+    ssl = False
+
+
+try:
+    create_connection = socket.create_connection
+except AttributeError:
+    def create_connection(address):
+        host, port = address
+        msg = "getaddrinfo returns an empty list"
+        sock = None
+        for res in socket.getaddrinfo(host, port, 0,
+                                      socket.SOCK_STREAM):
+            af, socktype, proto, _canonname, sa = res
+            try:
+                sock = socket.socket(af, socktype, proto)
+                logger.info("connect: (%s, %s)", host, port)
+                sock.connect(sa)
+            except socket.error, msg:
+                logger.info('connect fail: %s %s', host, port)
+                if sock:
+                    sock.close()
+                sock = None
+                continue
+            break
+        if not sock:
+            raise socket.error, msg
+        return sock
+
+if ssl:
+    wrap_socket = ssl.wrap_socket
+    CERT_NONE = ssl.CERT_NONE
+    CERT_OPTIONAL = ssl.CERT_OPTIONAL
+    CERT_REQUIRED = ssl.CERT_REQUIRED
+    PROTOCOL_SSLv2 = ssl.PROTOCOL_SSLv2
+    PROTOCOL_SSLv3 = ssl.PROTOCOL_SSLv3
+    PROTOCOL_SSLv23 = ssl.PROTOCOL_SSLv23
+    PROTOCOL_TLSv1 = ssl.PROTOCOL_TLSv1
+else:
+    class FakeSocket(httplib.FakeSocket):
+        """Socket wrapper that supports SSL.
+        """
+        # backport the behavior from Python 2.6, which is to busy wait
+        # on the socket instead of anything nice. Sigh.
+        # See http://bugs.python.org/issue3890 for more info.
+        def recv(self, buflen=1024, flags=0):
+            """ssl-aware wrapper around socket.recv
+            """
+            if flags != 0:
+                raise ValueError(
+                    "non-zero flags not allowed in calls to recv() on %s" %
+                    self.__class__)
+            while True:
+                try:
+                    return self._ssl.read(buflen)
+                except socket.sslerror, x:
+                    if x.args[0] == socket.SSL_ERROR_WANT_READ:
+                        continue
+                    else:
+                        raise x
+
+    PROTOCOL_SSLv2 = 0
+    PROTOCOL_SSLv3 = 1
+    PROTOCOL_SSLv23 = 2
+    PROTOCOL_TLSv1 = 3
+
+    CERT_NONE = 0
+    CERT_OPTIONAL = 1
+    CERT_REQUIRED = 2
+
+    def wrap_socket(sock, keyfile=None, certfile=None,
+                server_side=False, cert_reqs=CERT_NONE,
+                ssl_version=PROTOCOL_SSLv23, ca_certs=None,
+                do_handshake_on_connect=True,
+                suppress_ragged_eofs=True):
+        if cert_reqs != CERT_NONE and ca_certs:
+            raise CertificateValidationUnsupported(
+                'SSL certificate validation requires the ssl module'
+                '(included in Python 2.6 and later.)')
+        sslob = socket.ssl(sock)
+        # borrow httplib's workaround for no ssl.wrap_socket
+        sock = FakeSocket(sock, sslob)
+        return sock
+
+
+class CertificateValidationUnsupported(Exception):
+    """Exception raised when cert validation is requested but unavailable."""
+# no-check-code

mercurial/httpclient/tests/__init__.py

@@ -0,0 +1,1 b''
+# no-check-code

mercurial/httpclient/tests/simple_http_test.py

@@ -0,0 +1,366 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def _run_simple_test(self, host, server_data, expected_req, expected_data):
+        con = http.HTTPConnection(host)
+        con._connect()
+        con.sock.data = server_data
+        con.request('GET', '/')
+
+        self.assertStringEqual(expected_req, con.sock.sent)
+        self.assertEqual(expected_data, con.getresponse().read())
+
+    def test_broken_data_obj(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        self.assertRaises(http.BadRequestData,
+                          con.request, 'POST', '/', body=1)
+
+    def test_no_keepalive_http_1_0(self):
+        expected_request_one = """GET /remote/.hg/requires HTTP/1.1
+Host: localhost:9999
+range: bytes=0-
+accept-encoding: identity
+accept: application/mercurial-0.1
+user-agent: mercurial/proto-1.0
+
+""".replace('\n', '\r\n')
+        expected_response_headers = """HTTP/1.0 200 OK
+Server: SimpleHTTP/0.6 Python/2.6.1
+Date: Sun, 01 May 2011 13:56:57 GMT
+Content-type: application/octet-stream
+Content-Length: 33
+Last-Modified: Sun, 01 May 2011 13:56:56 GMT
+
+""".replace('\n', '\r\n')
+        expected_response_body = """revlogv1
+store
+fncache
+dotencode
+"""
+        con = http.HTTPConnection('localhost:9999')
+        con._connect()
+        con.sock.data = [expected_response_headers, expected_response_body]
+        con.request('GET', '/remote/.hg/requires',
+                    headers={'accept-encoding': 'identity',
+                             'range': 'bytes=0-',
+                             'accept': 'application/mercurial-0.1',
+                             'user-agent': 'mercurial/proto-1.0',
+                             })
+        self.assertStringEqual(expected_request_one, con.sock.sent)
+        self.assertEqual(con.sock.closed, False)
+        self.assertNotEqual(con.sock.data, [])
+        self.assert_(con.busy())
+        resp = con.getresponse()
+        self.assertStringEqual(resp.read(), expected_response_body)
+        self.failIf(con.busy())
+        self.assertEqual(con.sock, None)
+        self.assertEqual(resp.sock.data, [])
+        self.assert_(resp.sock.closed)
+
+    def test_multiline_header(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Multiline: Value\r\n',
+                         '  Rest of value\r\n',
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value\n Rest of value'],
+                         resp.headers.getheaders('multiline'))
+
+    def testSimpleRequest(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'MultiHeader: Value\r\n'
+                         'MultiHeader: Other Value\r\n'
+                         'MultiHeader: One More!\r\n'
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value', 'Other Value', 'One More!'],
+                         resp.headers.getheaders('multiheader'))
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testHeaderlessResponse(self):
+        con = http.HTTPConnection('1.2.3.4', use_ssl=False)
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual({}, dict(resp.headers))
+        self.assertEqual(resp.status, 200)
+
+    def testReadline(self):
+        con = http.HTTPConnection('1.2.3.4')
+        con._connect()
+        # make sure it trickles in one byte at a time
+        # so that we touch all the cases in readline
+        con.sock.data = list(''.join(
+            ['HTTP/1.1 200 OK\r\n',
+             'Server: BogusServer 1.0\r\n',
+             'Connection: Close\r\n',
+             '\r\n'
+             '1\n2\nabcdefg\n4\n5']))
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        con.request('GET', '/')
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        r = con.getresponse()
+        for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']:
+            actual = r.readline()
+            self.assertEqual(expected, actual,
+                             'Expected %r, got %r' % (expected, actual))
+
+    def testIPv6(self):
+        self._run_simple_test('[::1]:8221',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: [::1]:8221\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+        self._run_simple_test('::2',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: ::2\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+        self._run_simple_test('[::3]:443',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: ::3\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+
+    def doPost(self, con, expect_body, body_to_send='This is some POST data'):
+        con.request('POST', '/', body=body_to_send,
+                    expect_continue=True)
+        expected_req = ('POST / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'content-length: %d\r\n'
+                        'Expect: 100-Continue\r\n'
+                        'accept-encoding: identity\r\n\r\n' %
+                        len(body_to_send))
+        if expect_body:
+            expected_req += body_to_send
+        return expected_req
+
+    def testEarlyContinueResponse(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 18',
+                         '\r\n\r\n'
+                         "You can't do that."]
+        expected_req = self.doPost(con, expect_body=False)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertStringEqual(expected_req, sock.sent)
+        self.assertEqual("You can't do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, True)
+
+    def testDeniedAfterContinueTimeoutExpires(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 18\r\n',
+                     'Connection: close',
+                     '\r\n\r\n'
+                     "You can't do that."]
+        sock.read_wait_sentinel = 'Dear server, send response!'
+        sock.close_on_empty = True
+        # send enough data out that we'll chunk it into multiple
+        # blocks and the socket will close before we can send the
+        # whole request.
+        post_body = ('This is some POST data\n' * 1024 * 32 +
+                     'Dear server, send response!\n' +
+                     'This is some POST data\n' * 1024 * 32)
+        expected_req = self.doPost(con, expect_body=False,
+                                   body_to_send=post_body)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assert_('POST data\n' in sock.sent)
+        self.assert_('Dear server, send response!\n' in sock.sent)
+        # We expect not all of our data was sent.
+        self.assertNotEqual(sock.sent, expected_req)
+        self.assertEqual("You can't do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, True)
+
+    def testPostData(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.early_data = ['HTTP/1.1 100 Co', 'ntinue\r\n\r\n']
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        self.assertEqual("You can do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testServerWithoutContinue(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        self.assertEqual("You can do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testServerWithSlowContinue(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.data = ['HTTP/1.1 100 ', 'Continue\r\n\r\n',
+                     'HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        resp = con.getresponse()
+        self.assertEqual("You can do that.", resp.read())
+        self.assertEqual(200, resp.status)
+        self.assertEqual(sock.closed, False)
+
+    def testSlowConnection(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        # simulate one byte arriving at a time, to check for various
+        # corner cases
+        con.sock.data = list('HTTP/1.1 200 OK\r\n'
+                             'Server: BogusServer 1.0\r\n'
+                             'Content-Length: 10'
+                             '\r\n\r\n'
+                             '1234567890')
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual('1234567890', con.getresponse().read())
+
+    def testTimeout(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = []
+        con.request('GET', '/')
+        self.assertRaises(http.HTTPTimeoutException,
+                          con.getresponse)
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+# no-check-code

mercurial/httpclient/tests/test_bogus_responses.py

@@ -0,0 +1,68 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests against malformed responses.
+
+Server implementations that respond with only LF instead of CRLF have
+been observed. Checking against ones that use only CR is a hedge
+against that potential insanit.y
+"""
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def bogusEOL(self, eol):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK%s' % eol,
+                         'Server: BogusServer 1.0%s' % eol,
+                         'Content-Length: 10',
+                         eol * 2,
+                         '1234567890']
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual('1234567890', con.getresponse().read())
+
+    def testOnlyLinefeed(self):
+        self.bogusEOL('\n')
+
+    def testOnlyCarriageReturn(self):
+        self.bogusEOL('\r')
+# no-check-code

mercurial/httpclient/tests/test_chunked_transfer.py

@@ -0,0 +1,137 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import cStringIO
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+def chunkedblock(x, eol='\r\n'):
+    r"""Make a chunked transfer-encoding block.
+
+    >>> chunkedblock('hi')
+    '2\r\nhi\r\n'
+    >>> chunkedblock('hi' * 10)
+    '14\r\nhihihihihihihihihihi\r\n'
+    >>> chunkedblock('hi', eol='\n')
+    '2\nhi\n'
+    """
+    return ''.join((hex(len(x))[2:], eol, x, eol))
+
+
+class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase):
+    def testChunkedUpload(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'end-of-body'
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 6',
+                     '\r\n\r\n',
+                     "Thanks"]
+
+        zz = 'zz\n'
+        con.request('POST', '/', body=cStringIO.StringIO(
+            (zz * (0x8010 / 3)) + 'end-of-body'))
+        expected_req = ('POST / HTTP/1.1\r\n'
+                        'transfer-encoding: chunked\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+        expected_req += chunkedblock('zz\n' * (0x8000 / 3) + 'zz')
+        expected_req += chunkedblock(
+            '\n' + 'zz\n' * ((0x1b - len('end-of-body')) / 3) + 'end-of-body')
+        expected_req += '0\r\n\r\n'
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertStringEqual(expected_req, sock.sent)
+        self.assertEqual("Thanks", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testChunkedDownload(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'transfer-encoding: chunked',
+                     '\r\n\r\n',
+                     chunkedblock('hi '),
+                     chunkedblock('there'),
+                     chunkedblock(''),
+                     ]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there', con.getresponse().read())
+
+    def testChunkedDownloadBadEOL(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\n',
+                     'Server: BogusServer 1.0\n',
+                     'transfer-encoding: chunked',
+                     '\n\n',
+                     chunkedblock('hi ', eol='\n'),
+                     chunkedblock('there', eol='\n'),
+                     chunkedblock('', eol='\n'),
+                     ]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there', con.getresponse().read())
+
+    def testChunkedDownloadPartialChunkBadEOL(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\n',
+                     'Server: BogusServer 1.0\n',
+                     'transfer-encoding: chunked',
+                     '\n\n',
+                     chunkedblock('hi ', eol='\n'),
+                     ] + list(chunkedblock('there\n' * 5, eol='\n')) + [
+                         chunkedblock('', eol='\n')]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+                               con.getresponse().read())
+
+    def testChunkedDownloadPartialChunk(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'transfer-encoding: chunked',
+                     '\r\n\r\n',
+                     chunkedblock('hi '),
+                     ] + list(chunkedblock('there\n' * 5)) + [chunkedblock('')]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+                               con.getresponse().read())
+# no-check-code

mercurial/httpclient/tests/test_proxy_support.py

@@ -0,0 +1,132 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+import socket
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+def make_preloaded_socket(data):
+    """Make a socket pre-loaded with data so it can be read during connect.
+
+    Useful for https proxy tests because we have to read from the
+    socket during _connect rather than later on.
+    """
+    def s(*args, **kwargs):
+        sock = util.MockSocket(*args, **kwargs)
+        sock.data = data[:]
+        return sock
+    return s
+
+
+class ProxyHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def _run_simple_test(self, host, server_data, expected_req, expected_data):
+        con = http.HTTPConnection(host)
+        con._connect()
+        con.sock.data = server_data
+        con.request('GET', '/')
+
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual(expected_data, con.getresponse().read())
+
+    def testSimpleRequest(self):
+        con = http.HTTPConnection('1.2.3.4:80',
+                                  proxy_hostport=('magicproxy', 4242))
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'MultiHeader: Value\r\n'
+                         'MultiHeader: Other Value\r\n'
+                         'MultiHeader: One More!\r\n'
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET http://1.2.3.4/ HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+        self.assertStringEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value', 'Other Value', 'One More!'],
+                         resp.headers.getheaders('multiheader'))
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testSSLRequest(self):
+        con = http.HTTPConnection('1.2.3.4:443',
+                                  proxy_hostport=('magicproxy', 4242))
+        socket.socket = make_preloaded_socket(
+            ['HTTP/1.1 200 OK\r\n',
+             'Server: BogusServer 1.0\r\n',
+             'Content-Length: 10\r\n',
+             '\r\n'
+             '1234567890'])
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n'
+                        '\r\n'
+                        'GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+        self.assertStringEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual(resp.status, 200)
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testSSLProxyFailure(self):
+        con = http.HTTPConnection('1.2.3.4:443',
+                                  proxy_hostport=('magicproxy', 4242))
+        socket.socket = make_preloaded_socket(
+            ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'])
+        self.assertRaises(http.HTTPProxyConnectFailedException, con._connect)
+        self.assertRaises(http.HTTPProxyConnectFailedException,
+                          con.request, 'GET', '/')
+# no-check-code

mercurial/httpclient/tests/util.py

@@ -0,0 +1,160 b''
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import difflib
+import socket
+
+import http
+
+
+class MockSocket(object):
+    """Mock non-blocking socket object.
+
+    This is ONLY capable of mocking a nonblocking socket.
+
+    Attributes:
+      early_data: data to always send as soon as end of headers is seen
+      data: a list of strings to return on recv(), with the
+            assumption that the socket would block between each
+            string in the list.
+      read_wait_sentinel: data that must be written to the socket before
+                          beginning the response.
+      close_on_empty: If true, close the socket when it runs out of data
+                      for the client.
+    """
+    def __init__(self, af, socktype, proto):
+        self.af = af
+        self.socktype = socktype
+        self.proto = proto
+
+        self.early_data = []
+        self.data = []
+        self.remote_closed = self.closed = False
+        self.close_on_empty = False
+        self.sent = ''
+        self.read_wait_sentinel = http._END_HEADERS
+
+    def close(self):
+        self.closed = True
+
+    def connect(self, sa):
+        self.sa = sa
+
+    def setblocking(self, timeout):
+        assert timeout == 0
+
+    def recv(self, amt=-1):
+        if self.early_data:
+            datalist = self.early_data
+        elif not self.data:
+            return ''
+        else:
+            datalist = self.data
+        if amt == -1:
+            return datalist.pop(0)
+        data = datalist.pop(0)
+        if len(data) > amt:
+            datalist.insert(0, data[amt:])
+        if not self.data and not self.early_data and self.close_on_empty:
+            self.remote_closed = True
+        return data[:amt]
+
+    @property
+    def ready_for_read(self):
+        return ((self.early_data and http._END_HEADERS in self.sent)
+                or (self.read_wait_sentinel in self.sent and self.data)
+                or self.closed)
+
+    def send(self, data):
+        # this is a horrible mock, but nothing needs us to raise the
+        # correct exception yet
+        assert not self.closed, 'attempted to write to a closed socket'
+        assert not self.remote_closed, ('attempted to write to a'
+                                        ' socket closed by the server')
+        if len(data) > 8192:
+            data = data[:8192]
+        self.sent += data
+        return len(data)
+
+
+def mockselect(r, w, x, timeout=0):
+    """Simple mock for select()
+    """
+    readable = filter(lambda s: s.ready_for_read, r)
+    return readable, w[:], []
+
+
+def mocksslwrap(sock, keyfile=None, certfile=None,
+                server_side=False, cert_reqs=http.socketutil.CERT_NONE,
+                ssl_version=http.socketutil.PROTOCOL_SSLv23, ca_certs=None,
+                do_handshake_on_connect=True,
+                suppress_ragged_eofs=True):
+    return sock
+
+
+def mockgetaddrinfo(host, port, unused, streamtype):
+    assert unused == 0
+    assert streamtype == socket.SOCK_STREAM
+    if host.count('.') != 3:
+        host = '127.0.0.42'
+    return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '',
+             (host, port))]
+
+
+class HttpTestBase(object):
+    def setUp(self):
+        self.orig_socket = socket.socket
+        socket.socket = MockSocket
+
+        self.orig_getaddrinfo = socket.getaddrinfo
+        socket.getaddrinfo = mockgetaddrinfo
+
+        self.orig_select = http.select.select
+        http.select.select = mockselect
+
+        self.orig_sslwrap = http.socketutil.wrap_socket
+        http.socketutil.wrap_socket = mocksslwrap
+
+    def tearDown(self):
+        socket.socket = self.orig_socket
+        http.select.select = self.orig_select
+        http.socketutil.wrap_socket = self.orig_sslwrap
+        socket.getaddrinfo = self.orig_getaddrinfo
+
+    def assertStringEqual(self, l, r):
+        try:
+            self.assertEqual(l, r, ('failed string equality check, '
+                                    'see stdout for details'))
+        except:
+            add_nl = lambda li: map(lambda x: x + '\n', li)
+            print 'failed expectation:'
+            print ''.join(difflib.unified_diff(
+                add_nl(l.splitlines()), add_nl(r.splitlines()),
+                fromfile='expected', tofile='got'))
+            raise
+# no-check-code

setup.py

@@ -301,8 +301,9 b" cmdclass = {'build_mo': hgbuildmo,"
             'build_py': hgbuildpy,
             'install_scripts': hginstallscripts}
 
-packages = ['mercurial', 'mercurial.hgweb', 'hgext', 'hgext.convert',
-            'hgext.highlight', 'hgext.zeroconf']
+packages = ['mercurial', 'mercurial.hgweb',
+            'mercurial.httpclient', 'mercurial.httpclient.tests',
+            'hgext', 'hgext.convert', 'hgext.highlight', 'hgext.zeroconf']
 
 pymodules = []