Import new http library as mercurial.httpclient.

This is revision a4229f13c374 of
http://py-nonblocking-http.googlecode.com/ with a no-check-code
comment added to the end of each file using `for fi in $(hg manifest |
grep mercurial/httpclient/) ; echo '# no-check-code' >> $fi`.

mercurial/httpclient/__init__.py

@@ -0,0 +1,650
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Improved HTTP/1.1 client library
+
+This library contains an HTTPConnection which is similar to the one in
+httplib, but has several additional features:
+
+  * supports keepalives natively
+  * uses select() to block for incoming data
+  * notices when the server responds early to a request
+  * implements ssl inline instead of in a different class
+"""
+
+import cStringIO
+import errno
+import httplib
+import logging
+import rfc822
+import select
+import socket
+
+import socketutil
+
+logger = logging.getLogger(__name__)
+
+__all__ = ['HTTPConnection', 'HTTPResponse']
+
+HTTP_VER_1_0 = 'HTTP/1.0'
+HTTP_VER_1_1 = 'HTTP/1.1'
+
+_LEN_CLOSE_IS_END = -1
+
+OUTGOING_BUFFER_SIZE = 1 << 15
+INCOMING_BUFFER_SIZE = 1 << 20
+
+HDR_ACCEPT_ENCODING = 'accept-encoding'
+HDR_CONNECTION_CTRL = 'connection'
+HDR_CONTENT_LENGTH = 'content-length'
+HDR_XFER_ENCODING = 'transfer-encoding'
+
+XFER_ENCODING_CHUNKED = 'chunked'
+
+CONNECTION_CLOSE = 'close'
+
+EOL = '\r\n'
+_END_HEADERS = EOL * 2
+
+# Based on some searching around, 1 second seems like a reasonable
+# default here.
+TIMEOUT_ASSUME_CONTINUE = 1
+TIMEOUT_DEFAULT = None
+
+
+class HTTPResponse(object):
+    """Response from an HTTP server.
+
+    The response will continue to load as available. If you need the
+    complete response before continuing, check the .complete() method.
+    """
+    def __init__(self, sock, timeout):
+        self.sock = sock
+        self.raw_response = ''
+        self._body = None
+        self._headers_len = 0
+        self._content_len = 0
+        self.headers = None
+        self.will_close = False
+        self.status_line = ''
+        self.status = None
+        self.http_version = None
+        self.reason = None
+        self._chunked = False
+        self._chunked_done = False
+        self._chunked_until_next = 0
+        self._chunked_skip_bytes = 0
+        self._chunked_preloaded_block = None
+
+        self._read_location = 0
+        self._eol = EOL
+
+        self._timeout = timeout
+
+    @property
+    def _end_headers(self):
+        return self._eol * 2
+
+    def complete(self):
+        """Returns true if this response is completely loaded.
+        """
+        if self._chunked:
+            return self._chunked_done
+        if self._content_len == _LEN_CLOSE_IS_END:
+            return False
+        return self._body is not None and len(self._body) >= self._content_len
+
+    def readline(self):
+        """Read a single line from the response body.
+
+        This may block until either a line ending is found or the
+        response is complete.
+        """
+        eol = self._body.find('\n', self._read_location)
+        while eol == -1 and not self.complete():
+            self._select()
+            eol = self._body.find('\n', self._read_location)
+        if eol != -1:
+            eol += 1
+        else:
+            eol = len(self._body)
+        data = self._body[self._read_location:eol]
+        self._read_location = eol
+        return data
+
+    def read(self, length=None):
+        # if length is None, unbounded read
+        while (not self.complete()  # never select on a finished read
+               and (not length  # unbounded, so we wait for complete()
+                    or (self._read_location + length) > len(self._body))):
+            self._select()
+        if not length:
+            length = len(self._body) - self._read_location
+        elif len(self._body) < (self._read_location + length):
+            length = len(self._body) - self._read_location
+        r = self._body[self._read_location:self._read_location + length]
+        self._read_location += len(r)
+        if self.complete() and self.will_close:
+            self.sock.close()
+        return r
+
+    def _select(self):
+        r, _, _ = select.select([self.sock], [], [], self._timeout)
+        if not r:
+            # socket was not readable. If the response is not complete
+            # and we're not a _LEN_CLOSE_IS_END response, raise a timeout.
+            # If we are a _LEN_CLOSE_IS_END response and we have no data,
+            # raise a timeout.
+            if not (self.complete() or
+                    (self._content_len == _LEN_CLOSE_IS_END and self._body)):
+                logger.info('timed out with timeout of %s', self._timeout)
+                raise HTTPTimeoutException('timeout reading data')
+            logger.info('cl: %r body: %r', self._content_len, self._body)
+        data = self.sock.recv(INCOMING_BUFFER_SIZE)
+        logger.debug('response read %d data during _select', len(data))
+        if not data:
+            if not self.headers:
+                self._load_response(self._end_headers)
+                self._content_len = 0
+            elif self._content_len == _LEN_CLOSE_IS_END:
+                self._content_len = len(self._body)
+            return False
+        else:
+            self._load_response(data)
+            return True
+
+    def _chunked_parsedata(self, data):
+        if self._chunked_preloaded_block:
+            data = self._chunked_preloaded_block + data
+            self._chunked_preloaded_block = None
+        while data:
+            logger.debug('looping with %d data remaining', len(data))
+            # Slice out anything we should skip
+            if self._chunked_skip_bytes:
+                if len(data) <= self._chunked_skip_bytes:
+                    self._chunked_skip_bytes -= len(data)
+                    data = ''
+                    break
+                else:
+                    data = data[self._chunked_skip_bytes:]
+                    self._chunked_skip_bytes = 0
+
+            # determine how much is until the next chunk
+            if self._chunked_until_next:
+                amt = self._chunked_until_next
+                logger.debug('reading remaining %d of existing chunk', amt)
+                self._chunked_until_next = 0
+                body = data
+            else:
+                try:
+                    amt, body = data.split(self._eol, 1)
+                except ValueError:
+                    self._chunked_preloaded_block = data
+                    logger.debug('saving %r as a preloaded block for chunked',
+                                 self._chunked_preloaded_block)
+                    return
+                amt = int(amt, base=16)
+                logger.debug('reading chunk of length %d', amt)
+                if amt == 0:
+                    self._chunked_done = True
+
+            # read through end of what we have or the chunk
+            self._body += body[:amt]
+            if len(body) >= amt:
+                data = body[amt:]
+                self._chunked_skip_bytes = len(self._eol)
+            else:
+                self._chunked_until_next = amt - len(body)
+                self._chunked_skip_bytes = 0
+                data = ''
+
+    def _load_response(self, data):
+        if self._chunked:
+            self._chunked_parsedata(data)
+            return
+        elif self._body is not None:
+            self._body += data
+            return
+
+        # We haven't seen end of headers yet
+        self.raw_response += data
+        # This is a bogus server with bad line endings
+        if self._eol not in self.raw_response:
+            for bad_eol in ('\n', '\r'):
+                if (bad_eol in self.raw_response
+                    # verify that bad_eol is not the end of the incoming data
+                    # as this could be a response line that just got
+                    # split between \r and \n.
+                    and (self.raw_response.index(bad_eol) <
+                         (len(self.raw_response) - 1))):
+                    logger.info('bogus line endings detected, '
+                                'using %r for EOL', bad_eol)
+                    self._eol = bad_eol
+                    break
+        # exit early if not at end of headers
+        if self._end_headers not in self.raw_response or self.headers:
+            return
+
+        # handle 100-continue response
+        hdrs, body = self.raw_response.split(self._end_headers, 1)
+        http_ver, status = hdrs.split(' ', 1)
+        if status.startswith('100'):
+            self.raw_response = body
+            logger.debug('continue seen, setting body to %r', body)
+            return
+
+        # arriving here means we should parse response headers
+        # as all headers have arrived completely
+        hdrs, body = self.raw_response.split(self._end_headers, 1)
+        del self.raw_response
+        if self._eol in hdrs:
+            self.status_line, hdrs = hdrs.split(self._eol, 1)
+        else:
+            self.status_line = hdrs
+            hdrs = ''
+        # TODO HTTP < 1.0 support
+        (self.http_version, self.status,
+         self.reason) = self.status_line.split(' ', 2)
+        self.status = int(self.status)
+        if self._eol != EOL:
+            hdrs = hdrs.replace(self._eol, '\r\n')
+        headers = rfc822.Message(cStringIO.StringIO(hdrs))
+        if HDR_CONTENT_LENGTH in headers:
+            self._content_len = int(headers[HDR_CONTENT_LENGTH])
+        if self.http_version == HTTP_VER_1_0:
+            self.will_close = True
+        elif HDR_CONNECTION_CTRL in headers:
+            self.will_close = (
+                headers[HDR_CONNECTION_CTRL].lower() == CONNECTION_CLOSE)
+            if self._content_len == 0:
+                self._content_len = _LEN_CLOSE_IS_END
+        if (HDR_XFER_ENCODING in headers
+            and headers[HDR_XFER_ENCODING].lower() == XFER_ENCODING_CHUNKED):
+            self._body = ''
+            self._chunked_parsedata(body)
+            self._chunked = True
+        if self._body is None:
+            self._body = body
+        self.headers = headers
+
+
+class HTTPConnection(object):
+    """Connection to a single http server.
+
+    Supports 100-continue and keepalives natively. Uses select() for
+    non-blocking socket operations.
+    """
+    http_version = HTTP_VER_1_1
+    response_class = HTTPResponse
+
+    def __init__(self, host, port=None, use_ssl=None, ssl_validator=None,
+                 timeout=TIMEOUT_DEFAULT,
+                 continue_timeout=TIMEOUT_ASSUME_CONTINUE,
+                 proxy_hostport=None, **ssl_opts):
+        """Create a new HTTPConnection.
+
+        Args:
+          host: The host to which we'll connect.
+          port: Optional. The port over which we'll connect. Default 80 for
+                non-ssl, 443 for ssl.
+          use_ssl: Optional. Wether to use ssl. Defaults to False if port is
+                   not 443, true if port is 443.
+          ssl_validator: a function(socket) to validate the ssl cert
+          timeout: Optional. Connection timeout, default is TIMEOUT_DEFAULT.
+          continue_timeout: Optional. Timeout for waiting on an expected
+                   "100 Continue" response. Default is TIMEOUT_ASSUME_CONTINUE.
+          proxy_hostport: Optional. Tuple of (host, port) to use as an http
+                       proxy for the connection. Default is to not use a proxy.
+        """
+        if port is None and host.count(':') == 1 or ']:' in host:
+            host, port = host.rsplit(':', 1)
+            port = int(port)
+            if '[' in host:
+                host = host[1:-1]
+        if use_ssl is None and port is None:
+            use_ssl = False
+            port = 80
+        elif use_ssl is None:
+            use_ssl = (port == 443)
+        elif port is None:
+            port = (use_ssl and 443 or 80)
+        self.port = port
+        if use_ssl and not socketutil.have_ssl:
+            raise Exception('ssl requested but unavailable on this Python')
+        self.ssl = use_ssl
+        self.ssl_opts = ssl_opts
+        self._ssl_validator = ssl_validator
+        self.host = host
+        self.sock = None
+        self._current_response = None
+        self._current_response_taken = False
+        if proxy_hostport is None:
+            self._proxy_host = self._proxy_port = None
+        else:
+            self._proxy_host, self._proxy_port = proxy_hostport
+
+        self.timeout = timeout
+        self.continue_timeout = continue_timeout
+
+    def _connect(self):
+        """Connect to the host and port specified in __init__."""
+        if self.sock:
+            return
+        if self._proxy_host is not None:
+            logger.info('Connecting to http proxy %s:%s',
+                        self._proxy_host, self._proxy_port)
+            sock = socketutil.create_connection((self._proxy_host,
+                                                 self._proxy_port))
+            if self.ssl:
+                # TODO proxy header support
+                data = self.buildheaders('CONNECT', '%s:%d' % (self.host,
+                                                               self.port),
+                                         {}, HTTP_VER_1_0)
+                sock.send(data)
+                sock.setblocking(0)
+                r = self.response_class(sock, self.timeout)
+                timeout_exc = HTTPTimeoutException(
+                    'Timed out waiting for CONNECT response from proxy')
+                while not r.complete():
+                    try:
+                        if not r._select():
+                            raise timeout_exc
+                    except HTTPTimeoutException:
+                        # This raise/except pattern looks goofy, but
+                        # _select can raise the timeout as well as the
+                        # loop body. I wish it wasn't this convoluted,
+                        # but I don't have a better solution
+                        # immediately handy.
+                        raise timeout_exc
+                if r.status != 200:
+                    raise HTTPProxyConnectFailedException(
+                        'Proxy connection failed: %d %s' % (r.status,
+                                                            r.read()))
+                logger.info('CONNECT (for SSL) to %s:%s via proxy succeeded.',
+                            self.host, self.port)
+        else:
+            sock = socketutil.create_connection((self.host, self.port))
+        if self.ssl:
+            logger.debug('wrapping socket for ssl with options %r',
+                         self.ssl_opts)
+            sock = socketutil.wrap_socket(sock, **self.ssl_opts)
+            if self._ssl_validator:
+                self._ssl_validator(sock)
+        sock.setblocking(0)
+        self.sock = sock
+
+    def buildheaders(self, method, path, headers, http_ver):
+        if self.ssl and self.port == 443 or self.port == 80:
+            # default port for protocol, so leave it out
+            hdrhost = self.host
+        else:
+            # include nonstandard port in header
+            if ':' in self.host:  # must be IPv6
+                hdrhost = '[%s]:%d' % (self.host, self.port)
+            else:
+                hdrhost = '%s:%d' % (self.host, self.port)
+        if self._proxy_host and not self.ssl:
+            # When talking to a regular http proxy we must send the
+            # full URI, but in all other cases we must not (although
+            # technically RFC 2616 says servers must accept our
+            # request if we screw up, experimentally few do that
+            # correctly.)
+            assert path[0] == '/', 'path must start with a /'
+            path = 'http://%s%s' % (hdrhost, path)
+        outgoing = ['%s %s %s%s' % (method, path, http_ver, EOL)]
+        headers['host'] = ('Host', hdrhost)
+        headers[HDR_ACCEPT_ENCODING] = (HDR_ACCEPT_ENCODING, 'identity')
+        for hdr, val in headers.itervalues():
+            outgoing.append('%s: %s%s' % (hdr, val, EOL))
+        outgoing.append(EOL)
+        return ''.join(outgoing)
+
+    def close(self):
+        """Close the connection to the server.
+
+        This is a no-op if the connection is already closed. The
+        connection may automatically close if requessted by the server
+        or required by the nature of a response.
+        """
+        if self.sock is None:
+            return
+        self.sock.close()
+        self.sock = None
+        logger.info('closed connection to %s on %s', self.host, self.port)
+
+    def busy(self):
+        """Returns True if this connection object is currently in use.
+
+        If a response is still pending, this will return True, even if
+        the request has finished sending. In the future,
+        HTTPConnection may transparently juggle multiple connections
+        to the server, in which case this will be useful to detect if
+        any of those connections is ready for use.
+        """
+        cr = self._current_response
+        if cr is not None:
+            if self._current_response_taken:
+                if cr.will_close:
+                    self.sock = None
+                    self._current_response = None
+                    return False
+                elif cr.complete():
+                    self._current_response = None
+                    return False
+            return True
+        return False
+
+    def request(self, method, path, body=None, headers={},
+                expect_continue=False):
+        """Send a request to the server.
+
+        For increased flexibility, this does not return the response
+        object. Future versions of HTTPConnection that juggle multiple
+        sockets will be able to send (for example) 5 requests all at
+        once, and then let the requests arrive as data is
+        available. Use the `getresponse()` method to retrieve the
+        response.
+        """
+        if self.busy():
+            raise httplib.CannotSendRequest(
+                'Can not send another request before '
+                'current response is read!')
+        self._current_response_taken = False
+
+        logger.info('sending %s request for %s to %s on port %s',
+                    method, path, self.host, self.port)
+        hdrs = dict((k.lower(), (k, v)) for k, v in headers.iteritems())
+        if hdrs.get('expect', ('', ''))[1].lower() == '100-continue':
+            expect_continue = True
+        elif expect_continue:
+            hdrs['expect'] = ('Expect', '100-Continue')
+
+        chunked = False
+        if body and HDR_CONTENT_LENGTH not in hdrs:
+            if getattr(body, '__len__', False):
+                hdrs[HDR_CONTENT_LENGTH] = (HDR_CONTENT_LENGTH, len(body))
+            elif getattr(body, 'read', False):
+                hdrs[HDR_XFER_ENCODING] = (HDR_XFER_ENCODING,
+                                           XFER_ENCODING_CHUNKED)
+                chunked = True
+            else:
+                raise BadRequestData('body has no __len__() nor read()')
+
+        self._connect()
+        outgoing_headers = self.buildheaders(
+            method, path, hdrs, self.http_version)
+        response = None
+        first = True
+
+        def reconnect(where):
+            logger.info('reconnecting during %s', where)
+            self.close()
+            self._connect()
+
+        while ((outgoing_headers or body)
+               and not (response and response.complete())):
+            select_timeout = self.timeout
+            out = outgoing_headers or body
+            blocking_on_continue = False
+            if expect_continue and not outgoing_headers and not (
+                response and response.headers):
+                logger.info(
+                    'waiting up to %s seconds for'
+                    ' continue response from server',
+                    self.continue_timeout)
+                select_timeout = self.continue_timeout
+                blocking_on_continue = True
+                out = False
+            if out:
+                w = [self.sock]
+            else:
+                w = []
+            r, w, x = select.select([self.sock], w, [], select_timeout)
+            # if we were expecting a 100 continue and it's been long
+            # enough, just go ahead and assume it's ok. This is the
+            # recommended behavior from the RFC.
+            if r == w == x == []:
+                if blocking_on_continue:
+                    expect_continue = False
+                    logger.info('no response to continue expectation from '
+                                'server, optimistically sending request body')
+                else:
+                    raise HTTPTimeoutException('timeout sending data')
+            # TODO exceptional conditions with select? (what are those be?)
+            # TODO if the response is loading, must we finish sending at all?
+            #
+            # Certainly not if it's going to close the connection and/or
+            # the response is already done...I think.
+            was_first = first
+
+            # incoming data
+            if r:
+                try:
+                    data = r[0].recv(INCOMING_BUFFER_SIZE)
+                    if not data:
+                        logger.info('socket appears closed in read')
+                        outgoing_headers = body = None
+                        break
+                    if response is None:
+                        response = self.response_class(r[0], self.timeout)
+                    response._load_response(data)
+                    if (response._content_len == _LEN_CLOSE_IS_END
+                        and len(data) == 0):
+                        response._content_len = len(response._body)
+                    if response.complete():
+                        w = []
+                        response.will_close = True
+                except socket.error, e:
+                    if e[0] != errno.EPIPE and not was_first:
+                        raise
+                    if (response._content_len
+                        and response._content_len != _LEN_CLOSE_IS_END):
+                        outgoing_headers = sent_data + outgoing_headers
+                        reconnect('read')
+
+            # outgoing data
+            if w and out:
+                try:
+                    if getattr(out, 'read', False):
+                        data = out.read(OUTGOING_BUFFER_SIZE)
+                        if not data:
+                            continue
+                        if len(data) < OUTGOING_BUFFER_SIZE:
+                            if chunked:
+                                body = '0' + EOL + EOL
+                            else:
+                                body = None
+                        if chunked:
+                            out = hex(len(data))[2:] + EOL + data + EOL
+                        else:
+                            out = data
+                    amt = w[0].send(out)
+                except socket.error, e:
+                    if e[0] == socket.SSL_ERROR_WANT_WRITE and self.ssl:
+                        # This means that SSL hasn't flushed its buffer into
+                        # the socket yet.
+                        # TODO: find a way to block on ssl flushing its buffer
+                        # similar to selecting on a raw socket.
+                        continue
+                    elif (e[0] not in (errno.ECONNRESET, errno.EPIPE)
+                          and not first):
+                        raise
+                    reconnect('write')
+                    amt = self.sock.send(out)
+                logger.debug('sent %d', amt)
+                first = False
+                # stash data we think we sent in case the socket breaks
+                # when we read from it
+                if was_first:
+                    sent_data = out[:amt]
+                if out is body:
+                    body = out[amt:]
+                else:
+                    outgoing_headers = out[amt:]
+
+        # close if the server response said to or responded before eating
+        # the whole request
+        if response is None:
+            response = self.response_class(self.sock, self.timeout)
+        complete = response.complete()
+        data_left = bool(outgoing_headers or body)
+        if data_left:
+            logger.info('stopped sending request early, '
+                         'will close the socket to be safe.')
+            response.will_close = True
+        if response.will_close:
+            # The socket will be closed by the response, so we disown
+            # the socket
+            self.sock = None
+        self._current_response = response
+
+    def getresponse(self):
+        if self._current_response is None:
+            raise httplib.ResponseNotReady()
+        r = self._current_response
+        while r.headers is None:
+            r._select()
+        if r.complete() or r.will_close:
+            self.sock = None
+            self._current_response = None
+        else:
+            self._current_response_taken = True
+        return r
+
+
+class HTTPTimeoutException(httplib.HTTPException):
+    """A timeout occurred while waiting on the server."""
+
+
+class BadRequestData(httplib.HTTPException):
+    """Request body object has neither __len__ nor read."""
+
+
+class HTTPProxyConnectFailedException(httplib.HTTPException):
+    """Connecting to the HTTP proxy failed."""
+# no-check-code

mercurial/httpclient/socketutil.py

@@ -0,0 +1,134
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Abstraction to simplify socket use for Python < 2.6
+
+This will attempt to use the ssl module and the new
+socket.create_connection method, but fall back to the old
+methods if those are unavailable.
+"""
+import logging
+import socket
+
+logger = logging.getLogger(__name__)
+
+try:
+    import ssl
+    ssl.wrap_socket  # make demandimporters load the module
+    have_ssl = True
+except ImportError:
+    import httplib
+    import urllib2
+    have_ssl = getattr(urllib2, 'HTTPSHandler', False)
+    ssl = False
+
+
+try:
+    create_connection = socket.create_connection
+except AttributeError:
+    def create_connection(address):
+        host, port = address
+        msg = "getaddrinfo returns an empty list"
+        sock = None
+        for res in socket.getaddrinfo(host, port, 0,
+                                      socket.SOCK_STREAM):
+            af, socktype, proto, _canonname, sa = res
+            try:
+                sock = socket.socket(af, socktype, proto)
+                logger.info("connect: (%s, %s)", host, port)
+                sock.connect(sa)
+            except socket.error, msg:
+                logger.info('connect fail: %s %s', host, port)
+                if sock:
+                    sock.close()
+                sock = None
+                continue
+            break
+        if not sock:
+            raise socket.error, msg
+        return sock
+
+if ssl:
+    wrap_socket = ssl.wrap_socket
+    CERT_NONE = ssl.CERT_NONE
+    CERT_OPTIONAL = ssl.CERT_OPTIONAL
+    CERT_REQUIRED = ssl.CERT_REQUIRED
+    PROTOCOL_SSLv2 = ssl.PROTOCOL_SSLv2
+    PROTOCOL_SSLv3 = ssl.PROTOCOL_SSLv3
+    PROTOCOL_SSLv23 = ssl.PROTOCOL_SSLv23
+    PROTOCOL_TLSv1 = ssl.PROTOCOL_TLSv1
+else:
+    class FakeSocket(httplib.FakeSocket):
+        """Socket wrapper that supports SSL.
+        """
+        # backport the behavior from Python 2.6, which is to busy wait
+        # on the socket instead of anything nice. Sigh.
+        # See http://bugs.python.org/issue3890 for more info.
+        def recv(self, buflen=1024, flags=0):
+            """ssl-aware wrapper around socket.recv
+            """
+            if flags != 0:
+                raise ValueError(
+                    "non-zero flags not allowed in calls to recv() on %s" %
+                    self.__class__)
+            while True:
+                try:
+                    return self._ssl.read(buflen)
+                except socket.sslerror, x:
+                    if x.args[0] == socket.SSL_ERROR_WANT_READ:
+                        continue
+                    else:
+                        raise x
+
+    PROTOCOL_SSLv2 = 0
+    PROTOCOL_SSLv3 = 1
+    PROTOCOL_SSLv23 = 2
+    PROTOCOL_TLSv1 = 3
+
+    CERT_NONE = 0
+    CERT_OPTIONAL = 1
+    CERT_REQUIRED = 2
+
+    def wrap_socket(sock, keyfile=None, certfile=None,
+                server_side=False, cert_reqs=CERT_NONE,
+                ssl_version=PROTOCOL_SSLv23, ca_certs=None,
+                do_handshake_on_connect=True,
+                suppress_ragged_eofs=True):
+        if cert_reqs != CERT_NONE and ca_certs:
+            raise CertificateValidationUnsupported(
+                'SSL certificate validation requires the ssl module'
+                '(included in Python 2.6 and later.)')
+        sslob = socket.ssl(sock)
+        # borrow httplib's workaround for no ssl.wrap_socket
+        sock = FakeSocket(sock, sslob)
+        return sock
+
+
+class CertificateValidationUnsupported(Exception):
+    """Exception raised when cert validation is requested but unavailable."""
+# no-check-code

mercurial/httpclient/tests/__init__.py

@@ -0,0 +1,1
+# no-check-code

mercurial/httpclient/tests/simple_http_test.py

@@ -0,0 +1,366
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def _run_simple_test(self, host, server_data, expected_req, expected_data):
+        con = http.HTTPConnection(host)
+        con._connect()
+        con.sock.data = server_data
+        con.request('GET', '/')
+
+        self.assertStringEqual(expected_req, con.sock.sent)
+        self.assertEqual(expected_data, con.getresponse().read())
+
+    def test_broken_data_obj(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        self.assertRaises(http.BadRequestData,
+                          con.request, 'POST', '/', body=1)
+
+    def test_no_keepalive_http_1_0(self):
+        expected_request_one = """GET /remote/.hg/requires HTTP/1.1
+Host: localhost:9999
+range: bytes=0-
+accept-encoding: identity
+accept: application/mercurial-0.1
+user-agent: mercurial/proto-1.0
+
+""".replace('\n', '\r\n')
+        expected_response_headers = """HTTP/1.0 200 OK
+Server: SimpleHTTP/0.6 Python/2.6.1
+Date: Sun, 01 May 2011 13:56:57 GMT
+Content-type: application/octet-stream
+Content-Length: 33
+Last-Modified: Sun, 01 May 2011 13:56:56 GMT
+
+""".replace('\n', '\r\n')
+        expected_response_body = """revlogv1
+store
+fncache
+dotencode
+"""
+        con = http.HTTPConnection('localhost:9999')
+        con._connect()
+        con.sock.data = [expected_response_headers, expected_response_body]
+        con.request('GET', '/remote/.hg/requires',
+                    headers={'accept-encoding': 'identity',
+                             'range': 'bytes=0-',
+                             'accept': 'application/mercurial-0.1',
+                             'user-agent': 'mercurial/proto-1.0',
+                             })
+        self.assertStringEqual(expected_request_one, con.sock.sent)
+        self.assertEqual(con.sock.closed, False)
+        self.assertNotEqual(con.sock.data, [])
+        self.assert_(con.busy())
+        resp = con.getresponse()
+        self.assertStringEqual(resp.read(), expected_response_body)
+        self.failIf(con.busy())
+        self.assertEqual(con.sock, None)
+        self.assertEqual(resp.sock.data, [])
+        self.assert_(resp.sock.closed)
+
+    def test_multiline_header(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Multiline: Value\r\n',
+                         '  Rest of value\r\n',
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value\n Rest of value'],
+                         resp.headers.getheaders('multiline'))
+
+    def testSimpleRequest(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'MultiHeader: Value\r\n'
+                         'MultiHeader: Other Value\r\n'
+                         'MultiHeader: One More!\r\n'
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value', 'Other Value', 'One More!'],
+                         resp.headers.getheaders('multiheader'))
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testHeaderlessResponse(self):
+        con = http.HTTPConnection('1.2.3.4', use_ssl=False)
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual({}, dict(resp.headers))
+        self.assertEqual(resp.status, 200)
+
+    def testReadline(self):
+        con = http.HTTPConnection('1.2.3.4')
+        con._connect()
+        # make sure it trickles in one byte at a time
+        # so that we touch all the cases in readline
+        con.sock.data = list(''.join(
+            ['HTTP/1.1 200 OK\r\n',
+             'Server: BogusServer 1.0\r\n',
+             'Connection: Close\r\n',
+             '\r\n'
+             '1\n2\nabcdefg\n4\n5']))
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        con.request('GET', '/')
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        r = con.getresponse()
+        for expected in ['1\n', '2\n', 'abcdefg\n', '4\n', '5']:
+            actual = r.readline()
+            self.assertEqual(expected, actual,
+                             'Expected %r, got %r' % (expected, actual))
+
+    def testIPv6(self):
+        self._run_simple_test('[::1]:8221',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: [::1]:8221\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+        self._run_simple_test('::2',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: ::2\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+        self._run_simple_test('[::3]:443',
+                        ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10',
+                         '\r\n\r\n'
+                         '1234567890'],
+                        ('GET / HTTP/1.1\r\n'
+                         'Host: ::3\r\n'
+                         'accept-encoding: identity\r\n\r\n'),
+                        '1234567890')
+
+    def doPost(self, con, expect_body, body_to_send='This is some POST data'):
+        con.request('POST', '/', body=body_to_send,
+                    expect_continue=True)
+        expected_req = ('POST / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'content-length: %d\r\n'
+                        'Expect: 100-Continue\r\n'
+                        'accept-encoding: identity\r\n\r\n' %
+                        len(body_to_send))
+        if expect_body:
+            expected_req += body_to_send
+        return expected_req
+
+    def testEarlyContinueResponse(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 18',
+                         '\r\n\r\n'
+                         "You can't do that."]
+        expected_req = self.doPost(con, expect_body=False)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertStringEqual(expected_req, sock.sent)
+        self.assertEqual("You can't do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, True)
+
+    def testDeniedAfterContinueTimeoutExpires(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 403 Forbidden\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 18\r\n',
+                     'Connection: close',
+                     '\r\n\r\n'
+                     "You can't do that."]
+        sock.read_wait_sentinel = 'Dear server, send response!'
+        sock.close_on_empty = True
+        # send enough data out that we'll chunk it into multiple
+        # blocks and the socket will close before we can send the
+        # whole request.
+        post_body = ('This is some POST data\n' * 1024 * 32 +
+                     'Dear server, send response!\n' +
+                     'This is some POST data\n' * 1024 * 32)
+        expected_req = self.doPost(con, expect_body=False,
+                                   body_to_send=post_body)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assert_('POST data\n' in sock.sent)
+        self.assert_('Dear server, send response!\n' in sock.sent)
+        # We expect not all of our data was sent.
+        self.assertNotEqual(sock.sent, expected_req)
+        self.assertEqual("You can't do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, True)
+
+    def testPostData(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.early_data = ['HTTP/1.1 100 Co', 'ntinue\r\n\r\n']
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        self.assertEqual("You can do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testServerWithoutContinue(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        self.assertEqual("You can do that.", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testServerWithSlowContinue(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'POST data'
+        sock.data = ['HTTP/1.1 100 ', 'Continue\r\n\r\n',
+                     'HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 16',
+                     '\r\n\r\n',
+                     "You can do that."]
+        expected_req = self.doPost(con, expect_body=True)
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertEqual(expected_req, sock.sent)
+        resp = con.getresponse()
+        self.assertEqual("You can do that.", resp.read())
+        self.assertEqual(200, resp.status)
+        self.assertEqual(sock.closed, False)
+
+    def testSlowConnection(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        # simulate one byte arriving at a time, to check for various
+        # corner cases
+        con.sock.data = list('HTTP/1.1 200 OK\r\n'
+                             'Server: BogusServer 1.0\r\n'
+                             'Content-Length: 10'
+                             '\r\n\r\n'
+                             '1234567890')
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual('1234567890', con.getresponse().read())
+
+    def testTimeout(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = []
+        con.request('GET', '/')
+        self.assertRaises(http.HTTPTimeoutException,
+                          con.getresponse)
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+# no-check-code

mercurial/httpclient/tests/test_bogus_responses.py

@@ -0,0 +1,68
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Tests against malformed responses.
+
+Server implementations that respond with only LF instead of CRLF have
+been observed. Checking against ones that use only CR is a hedge
+against that potential insanit.y
+"""
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+class SimpleHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def bogusEOL(self, eol):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK%s' % eol,
+                         'Server: BogusServer 1.0%s' % eol,
+                         'Content-Length: 10',
+                         eol * 2,
+                         '1234567890']
+        con.request('GET', '/')
+
+        expected_req = ('GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('1.2.3.4', 80), con.sock.sa)
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual('1234567890', con.getresponse().read())
+
+    def testOnlyLinefeed(self):
+        self.bogusEOL('\n')
+
+    def testOnlyCarriageReturn(self):
+        self.bogusEOL('\r')
+# no-check-code

mercurial/httpclient/tests/test_chunked_transfer.py

@@ -0,0 +1,137
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import cStringIO
+import unittest
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+def chunkedblock(x, eol='\r\n'):
+    r"""Make a chunked transfer-encoding block.
+
+    >>> chunkedblock('hi')
+    '2\r\nhi\r\n'
+    >>> chunkedblock('hi' * 10)
+    '14\r\nhihihihihihihihihihi\r\n'
+    >>> chunkedblock('hi', eol='\n')
+    '2\nhi\n'
+    """
+    return ''.join((hex(len(x))[2:], eol, x, eol))
+
+
+class ChunkedTransferTest(util.HttpTestBase, unittest.TestCase):
+    def testChunkedUpload(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.read_wait_sentinel = 'end-of-body'
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'Content-Length: 6',
+                     '\r\n\r\n',
+                     "Thanks"]
+
+        zz = 'zz\n'
+        con.request('POST', '/', body=cStringIO.StringIO(
+            (zz * (0x8010 / 3)) + 'end-of-body'))
+        expected_req = ('POST / HTTP/1.1\r\n'
+                        'transfer-encoding: chunked\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+        expected_req += chunkedblock('zz\n' * (0x8000 / 3) + 'zz')
+        expected_req += chunkedblock(
+            '\n' + 'zz\n' * ((0x1b - len('end-of-body')) / 3) + 'end-of-body')
+        expected_req += '0\r\n\r\n'
+        self.assertEqual(('1.2.3.4', 80), sock.sa)
+        self.assertStringEqual(expected_req, sock.sent)
+        self.assertEqual("Thanks", con.getresponse().read())
+        self.assertEqual(sock.closed, False)
+
+    def testChunkedDownload(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'transfer-encoding: chunked',
+                     '\r\n\r\n',
+                     chunkedblock('hi '),
+                     chunkedblock('there'),
+                     chunkedblock(''),
+                     ]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there', con.getresponse().read())
+
+    def testChunkedDownloadBadEOL(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\n',
+                     'Server: BogusServer 1.0\n',
+                     'transfer-encoding: chunked',
+                     '\n\n',
+                     chunkedblock('hi ', eol='\n'),
+                     chunkedblock('there', eol='\n'),
+                     chunkedblock('', eol='\n'),
+                     ]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there', con.getresponse().read())
+
+    def testChunkedDownloadPartialChunkBadEOL(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\n',
+                     'Server: BogusServer 1.0\n',
+                     'transfer-encoding: chunked',
+                     '\n\n',
+                     chunkedblock('hi ', eol='\n'),
+                     ] + list(chunkedblock('there\n' * 5, eol='\n')) + [
+                         chunkedblock('', eol='\n')]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+                               con.getresponse().read())
+
+    def testChunkedDownloadPartialChunk(self):
+        con = http.HTTPConnection('1.2.3.4:80')
+        con._connect()
+        sock = con.sock
+        sock.data = ['HTTP/1.1 200 OK\r\n',
+                     'Server: BogusServer 1.0\r\n',
+                     'transfer-encoding: chunked',
+                     '\r\n\r\n',
+                     chunkedblock('hi '),
+                     ] + list(chunkedblock('there\n' * 5)) + [chunkedblock('')]
+        con.request('GET', '/')
+        self.assertStringEqual('hi there\nthere\nthere\nthere\nthere\n',
+                               con.getresponse().read())
+# no-check-code

mercurial/httpclient/tests/test_proxy_support.py

@@ -0,0 +1,132
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import unittest
+import socket
+
+import http
+
+# relative import to ease embedding the library
+import util
+
+
+def make_preloaded_socket(data):
+    """Make a socket pre-loaded with data so it can be read during connect.
+
+    Useful for https proxy tests because we have to read from the
+    socket during _connect rather than later on.
+    """
+    def s(*args, **kwargs):
+        sock = util.MockSocket(*args, **kwargs)
+        sock.data = data[:]
+        return sock
+    return s
+
+
+class ProxyHttpTest(util.HttpTestBase, unittest.TestCase):
+
+    def _run_simple_test(self, host, server_data, expected_req, expected_data):
+        con = http.HTTPConnection(host)
+        con._connect()
+        con.sock.data = server_data
+        con.request('GET', '/')
+
+        self.assertEqual(expected_req, con.sock.sent)
+        self.assertEqual(expected_data, con.getresponse().read())
+
+    def testSimpleRequest(self):
+        con = http.HTTPConnection('1.2.3.4:80',
+                                  proxy_hostport=('magicproxy', 4242))
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'MultiHeader: Value\r\n'
+                         'MultiHeader: Other Value\r\n'
+                         'MultiHeader: One More!\r\n'
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('GET http://1.2.3.4/ HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+        self.assertStringEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['Value', 'Other Value', 'One More!'],
+                         resp.headers.getheaders('multiheader'))
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testSSLRequest(self):
+        con = http.HTTPConnection('1.2.3.4:443',
+                                  proxy_hostport=('magicproxy', 4242))
+        socket.socket = make_preloaded_socket(
+            ['HTTP/1.1 200 OK\r\n',
+             'Server: BogusServer 1.0\r\n',
+             'Content-Length: 10\r\n',
+             '\r\n'
+             '1234567890'])
+        con._connect()
+        con.sock.data = ['HTTP/1.1 200 OK\r\n',
+                         'Server: BogusServer 1.0\r\n',
+                         'Content-Length: 10\r\n',
+                         '\r\n'
+                         '1234567890'
+                         ]
+        con.request('GET', '/')
+
+        expected_req = ('CONNECT 1.2.3.4:443 HTTP/1.0\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n'
+                        '\r\n'
+                        'GET / HTTP/1.1\r\n'
+                        'Host: 1.2.3.4\r\n'
+                        'accept-encoding: identity\r\n\r\n')
+
+        self.assertEqual(('127.0.0.42', 4242), con.sock.sa)
+        self.assertStringEqual(expected_req, con.sock.sent)
+        resp = con.getresponse()
+        self.assertEqual(resp.status, 200)
+        self.assertEqual('1234567890', resp.read())
+        self.assertEqual(['BogusServer 1.0'],
+                         resp.headers.getheaders('server'))
+
+    def testSSLProxyFailure(self):
+        con = http.HTTPConnection('1.2.3.4:443',
+                                  proxy_hostport=('magicproxy', 4242))
+        socket.socket = make_preloaded_socket(
+            ['HTTP/1.1 407 Proxy Authentication Required\r\n\r\n'])
+        self.assertRaises(http.HTTPProxyConnectFailedException, con._connect)
+        self.assertRaises(http.HTTPProxyConnectFailedException,
+                          con.request, 'GET', '/')
+# no-check-code

mercurial/httpclient/tests/util.py

@@ -0,0 +1,160
+# Copyright 2010, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import difflib
+import socket
+
+import http
+
+
+class MockSocket(object):
+    """Mock non-blocking socket object.
+
+    This is ONLY capable of mocking a nonblocking socket.
+
+    Attributes:
+      early_data: data to always send as soon as end of headers is seen
+      data: a list of strings to return on recv(), with the
+            assumption that the socket would block between each
+            string in the list.
+      read_wait_sentinel: data that must be written to the socket before
+                          beginning the response.
+      close_on_empty: If true, close the socket when it runs out of data
+                      for the client.
+    """
+    def __init__(self, af, socktype, proto):
+        self.af = af
+        self.socktype = socktype
+        self.proto = proto
+
+        self.early_data = []
+        self.data = []
+        self.remote_closed = self.closed = False
+        self.close_on_empty = False
+        self.sent = ''
+        self.read_wait_sentinel = http._END_HEADERS
+
+    def close(self):
+        self.closed = True
+
+    def connect(self, sa):
+        self.sa = sa
+
+    def setblocking(self, timeout):
+        assert timeout == 0
+
+    def recv(self, amt=-1):
+        if self.early_data:
+            datalist = self.early_data
+        elif not self.data:
+            return ''
+        else:
+            datalist = self.data
+        if amt == -1:
+            return datalist.pop(0)
+        data = datalist.pop(0)
+        if len(data) > amt:
+            datalist.insert(0, data[amt:])
+        if not self.data and not self.early_data and self.close_on_empty:
+            self.remote_closed = True
+        return data[:amt]
+
+    @property
+    def ready_for_read(self):
+        return ((self.early_data and http._END_HEADERS in self.sent)
+                or (self.read_wait_sentinel in self.sent and self.data)
+                or self.closed)
+
+    def send(self, data):
+        # this is a horrible mock, but nothing needs us to raise the
+        # correct exception yet
+        assert not self.closed, 'attempted to write to a closed socket'
+        assert not self.remote_closed, ('attempted to write to a'
+                                        ' socket closed by the server')
+        if len(data) > 8192:
+            data = data[:8192]
+        self.sent += data
+        return len(data)
+
+
+def mockselect(r, w, x, timeout=0):
+    """Simple mock for select()
+    """
+    readable = filter(lambda s: s.ready_for_read, r)
+    return readable, w[:], []
+
+
+def mocksslwrap(sock, keyfile=None, certfile=None,
+                server_side=False, cert_reqs=http.socketutil.CERT_NONE,
+                ssl_version=http.socketutil.PROTOCOL_SSLv23, ca_certs=None,
+                do_handshake_on_connect=True,
+                suppress_ragged_eofs=True):
+    return sock
+
+
+def mockgetaddrinfo(host, port, unused, streamtype):
+    assert unused == 0
+    assert streamtype == socket.SOCK_STREAM
+    if host.count('.') != 3:
+        host = '127.0.0.42'
+    return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, '',
+             (host, port))]
+
+
+class HttpTestBase(object):
+    def setUp(self):
+        self.orig_socket = socket.socket
+        socket.socket = MockSocket
+
+        self.orig_getaddrinfo = socket.getaddrinfo
+        socket.getaddrinfo = mockgetaddrinfo
+
+        self.orig_select = http.select.select
+        http.select.select = mockselect
+
+        self.orig_sslwrap = http.socketutil.wrap_socket
+        http.socketutil.wrap_socket = mocksslwrap
+
+    def tearDown(self):
+        socket.socket = self.orig_socket
+        http.select.select = self.orig_select
+        http.socketutil.wrap_socket = self.orig_sslwrap
+        socket.getaddrinfo = self.orig_getaddrinfo
+
+    def assertStringEqual(self, l, r):
+        try:
+            self.assertEqual(l, r, ('failed string equality check, '
+                                    'see stdout for details'))
+        except:
+            add_nl = lambda li: map(lambda x: x + '\n', li)
+            print 'failed expectation:'
+            print ''.join(difflib.unified_diff(
+                add_nl(l.splitlines()), add_nl(r.splitlines()),
+                fromfile='expected', tofile='got'))
+            raise
+# no-check-code

setup.py

@@ -1,399 +1,400
 #
 # This is the mercurial setup script.
 #
 # 'python setup.py install', or
 # 'python setup.py --help' for more options
 
 import sys
 if not hasattr(sys, 'version_info') or sys.version_info < (2, 4, 0, 'final'):
     raise SystemExit("Mercurial requires Python 2.4 or later.")
 
 if sys.version_info[0] >= 3:
     def b(s):
         '''A helper function to emulate 2.6+ bytes literals using string
         literals.'''
         return s.encode('latin1')
 else:
     def b(s):
         '''A helper function to emulate 2.6+ bytes literals using string
         literals.'''
         return s
 
 # Solaris Python packaging brain damage
 try:
     import hashlib
     sha = hashlib.sha1()
 except:
     try:
         import sha
     except:
         raise SystemExit(
             "Couldn't import standard hashlib (incomplete Python install).")
 
 try:
     import zlib
 except:
     raise SystemExit(
         "Couldn't import standard zlib (incomplete Python install).")
 
 try:
     import bz2
 except:
     raise SystemExit(
         "Couldn't import standard bz2 (incomplete Python install).")
 
 import os, subprocess, time
 import shutil
 import tempfile
 from distutils import log
 from distutils.core import setup, Extension
 from distutils.dist import Distribution
 from distutils.command.build import build
 from distutils.command.build_ext import build_ext
 from distutils.command.build_py import build_py
 from distutils.command.install_scripts import install_scripts
 from distutils.spawn import spawn, find_executable
 from distutils.ccompiler import new_compiler
 from distutils.errors import CCompilerError
 from distutils.sysconfig import get_python_inc
 from distutils.version import StrictVersion
 
 scripts = ['hg']
 if os.name == 'nt':
     scripts.append('contrib/win32/hg.bat')
 
 # simplified version of distutils.ccompiler.CCompiler.has_function
 # that actually removes its temporary files.
 def hasfunction(cc, funcname):
     tmpdir = tempfile.mkdtemp(prefix='hg-install-')
     devnull = oldstderr = None
     try:
         try:
             fname = os.path.join(tmpdir, 'funcname.c')
             f = open(fname, 'w')
             f.write('int main(void) {\n')
             f.write('    %s();\n' % funcname)
             f.write('}\n')
             f.close()
             # Redirect stderr to /dev/null to hide any error messages
             # from the compiler.
             # This will have to be changed if we ever have to check
             # for a function on Windows.
             devnull = open('/dev/null', 'w')
             oldstderr = os.dup(sys.stderr.fileno())
             os.dup2(devnull.fileno(), sys.stderr.fileno())
             objects = cc.compile([fname], output_dir=tmpdir)
             cc.link_executable(objects, os.path.join(tmpdir, "a.out"))
         except:
             return False
         return True
     finally:
         if oldstderr is not None:
             os.dup2(oldstderr, sys.stderr.fileno())
         if devnull is not None:
             devnull.close()
         shutil.rmtree(tmpdir)
 
 # py2exe needs to be installed to work
 try:
     import py2exe
     py2exeloaded = True
 except ImportError:
     py2exeloaded = False
 
 def runcmd(cmd, env):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, env=env)
     out, err = p.communicate()
     return out, err
 
 def runhg(cmd, env):
     out, err = runcmd(cmd, env)
     # If root is executing setup.py, but the repository is owned by
     # another user (as in "sudo python setup.py install") we will get
     # trust warnings since the .hg/hgrc file is untrusted. That is
     # fine, we don't want to load it anyway.  Python may warn about
     # a missing __init__.py in mercurial/locale, we also ignore that.
     err = [e for e in err.splitlines()
            if not e.startswith(b('Not trusting file')) \
               and not e.startswith(b('warning: Not importing'))]
     if err:
         return ''
     return out
 
 version = ''
 
 if os.path.isdir('.hg'):
     # Execute hg out of this directory with a custom environment which
     # includes the pure Python modules in mercurial/pure. We also take
     # care to not use any hgrc files and do no localization.
     pypath = ['mercurial', os.path.join('mercurial', 'pure')]
     env = {'PYTHONPATH': os.pathsep.join(pypath),
            'HGRCPATH': '',
            'LANGUAGE': 'C'}
     if 'LD_LIBRARY_PATH' in os.environ:
         env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
     if 'SystemRoot' in os.environ:
         # Copy SystemRoot into the custom environment for Python 2.6
         # under Windows. Otherwise, the subprocess will fail with
         # error 0xc0150004. See: http://bugs.python.org/issue3440
         env['SystemRoot'] = os.environ['SystemRoot']
     cmd = [sys.executable, 'hg', 'id', '-i', '-t']
     l = runhg(cmd, env).split()
     while len(l) > 1 and l[-1][0].isalpha(): # remove non-numbered tags
         l.pop()
     if len(l) > 1: # tag found
         version = l[-1]
         if l[0].endswith('+'): # propagate the dirty status to the tag
             version += '+'
     elif len(l) == 1: # no tag found
         cmd = [sys.executable, 'hg', 'parents', '--template',
                '{latesttag}+{latesttagdistance}-']
         version = runhg(cmd, env) + l[0]
     if version.endswith('+'):
         version += time.strftime('%Y%m%d')
 elif os.path.exists('.hg_archival.txt'):
     kw = dict([[t.strip() for t in l.split(':', 1)]
                for l in open('.hg_archival.txt')])
     if 'tag' in kw:
         version =  kw['tag']
     elif 'latesttag' in kw:
         version = '%(latesttag)s+%(latesttagdistance)s-%(node).12s' % kw
     else:
         version = kw.get('node', '')[:12]
 
 if version:
     f = open("mercurial/__version__.py", "w")
     f.write('# this file is autogenerated by setup.py\n')
     f.write('version = "%s"\n' % version)
     f.close()
 
 
 try:
     from mercurial import __version__
     version = __version__.version
 except ImportError:
     version = 'unknown'
 
 class hgbuildmo(build):
 
     description = "build translations (.mo files)"
 
     def run(self):
         if not find_executable('msgfmt'):
             self.warn("could not find msgfmt executable, no translations "
                      "will be built")
             return
 
         podir = 'i18n'
         if not os.path.isdir(podir):
             self.warn("could not find %s/ directory" % podir)
             return
 
         join = os.path.join
         for po in os.listdir(podir):
             if not po.endswith('.po'):
                 continue
             pofile = join(podir, po)
             modir = join('locale', po[:-3], 'LC_MESSAGES')
             mofile = join(modir, 'hg.mo')
             mobuildfile = join('mercurial', mofile)
             cmd = ['msgfmt', '-v', '-o', mobuildfile, pofile]
             if sys.platform != 'sunos5':
                 # msgfmt on Solaris does not know about -c
                 cmd.append('-c')
             self.mkpath(join('mercurial', modir))
             self.make_file([pofile], mobuildfile, spawn, (cmd,))
 
 
 # Insert hgbuildmo first so that files in mercurial/locale/ are found
 # when build_py is run next.
 build.sub_commands.insert(0, ('build_mo', None))
 
 Distribution.pure = 0
 Distribution.global_options.append(('pure', None, "use pure (slow) Python "
                                     "code instead of C extensions"))
 
 class hgbuildext(build_ext):
 
     def build_extension(self, ext):
         try:
             build_ext.build_extension(self, ext)
         except CCompilerError:
             if not getattr(ext, 'optional', False):
                 raise
             log.warn("Failed to build optional extension '%s' (skipping)",
                      ext.name)
 
 class hgbuildpy(build_py):
 
     def finalize_options(self):
         build_py.finalize_options(self)
 
         if self.distribution.pure:
             if self.py_modules is None:
                 self.py_modules = []
             for ext in self.distribution.ext_modules:
                 if ext.name.startswith("mercurial."):
                     self.py_modules.append("mercurial.pure.%s" % ext.name[10:])
             self.distribution.ext_modules = []
         else:
             if not os.path.exists(os.path.join(get_python_inc(), 'Python.h')):
                 raise SystemExit("Python headers are required to build Mercurial")
 
     def find_modules(self):
         modules = build_py.find_modules(self)
         for module in modules:
             if module[0] == "mercurial.pure":
                 if module[1] != "__init__":
                     yield ("mercurial", module[1], module[2])
             else:
                 yield module
 
 class hginstallscripts(install_scripts):
     '''
     This is a specialization of install_scripts that replaces the @LIBDIR@ with
     the configured directory for modules. If possible, the path is made relative
     to the directory for scripts.
     '''
 
     def initialize_options(self):
         install_scripts.initialize_options(self)
 
         self.install_lib = None
 
     def finalize_options(self):
         install_scripts.finalize_options(self)
         self.set_undefined_options('install',
                                    ('install_lib', 'install_lib'))
 
     def run(self):
         install_scripts.run(self)
 
         if (os.path.splitdrive(self.install_dir)[0] !=
             os.path.splitdrive(self.install_lib)[0]):
             # can't make relative paths from one drive to another, so use an
             # absolute path instead
             libdir = self.install_lib
         else:
             common = os.path.commonprefix((self.install_dir, self.install_lib))
             rest = self.install_dir[len(common):]
             uplevel = len([n for n in os.path.split(rest) if n])
 
             libdir =  uplevel * ('..' + os.sep) + self.install_lib[len(common):]
 
         for outfile in self.outfiles:
             fp = open(outfile, 'rb')
             data = fp.read()
             fp.close()
 
             # skip binary files
             if '\0' in data:
                 continue
 
             data = data.replace('@LIBDIR@', libdir.encode('string_escape'))
             fp = open(outfile, 'wb')
             fp.write(data)
             fp.close()
 
 cmdclass = {'build_mo': hgbuildmo,
             'build_ext': hgbuildext,
             'build_py': hgbuildpy,
             'install_scripts': hginstallscripts}
 
-packages = ['mercurial', 'mercurial.hgweb', 'hgext', 'hgext.convert',
-            'hgext.highlight', 'hgext.zeroconf']
+packages = ['mercurial', 'mercurial.hgweb',
+            'mercurial.httpclient', 'mercurial.httpclient.tests',
+            'hgext', 'hgext.convert', 'hgext.highlight', 'hgext.zeroconf']
 
 pymodules = []
 
 extmodules = [
     Extension('mercurial.base85', ['mercurial/base85.c']),
     Extension('mercurial.bdiff', ['mercurial/bdiff.c']),
     Extension('mercurial.diffhelpers', ['mercurial/diffhelpers.c']),
     Extension('mercurial.mpatch', ['mercurial/mpatch.c']),
     Extension('mercurial.parsers', ['mercurial/parsers.c']),
     ]
 
 osutil_ldflags = []
 
 if sys.platform == 'darwin':
     osutil_ldflags += ['-framework', 'ApplicationServices']
 
 # disable osutil.c under windows + python 2.4 (issue1364)
 if sys.platform == 'win32' and sys.version_info < (2, 5, 0, 'final'):
     pymodules.append('mercurial.pure.osutil')
 else:
     extmodules.append(Extension('mercurial.osutil', ['mercurial/osutil.c'],
                                 extra_link_args=osutil_ldflags))
 
 if sys.platform == 'linux2' and os.uname()[2] > '2.6':
     # The inotify extension is only usable with Linux 2.6 kernels.
     # You also need a reasonably recent C library.
     # In any case, if it fails to build the error will be skipped ('optional').
     cc = new_compiler()
     if hasfunction(cc, 'inotify_add_watch'):
         inotify = Extension('hgext.inotify.linux._inotify',
                             ['hgext/inotify/linux/_inotify.c'],
                             ['mercurial'])
         inotify.optional = True
         extmodules.append(inotify)
         packages.extend(['hgext.inotify', 'hgext.inotify.linux'])
 
 packagedata = {'mercurial': ['locale/*/LC_MESSAGES/hg.mo',
                              'help/*.txt']}
 
 def ordinarypath(p):
     return p and p[0] != '.' and p[-1] != '~'
 
 for root in ('templates',):
     for curdir, dirs, files in os.walk(os.path.join('mercurial', root)):
         curdir = curdir.split(os.sep, 1)[1]
         dirs[:] = filter(ordinarypath, dirs)
         for f in filter(ordinarypath, files):
             f = os.path.join(curdir, f)
             packagedata['mercurial'].append(f)
 
 datafiles = []
 setupversion = version
 extra = {}
 
 if py2exeloaded:
     extra['console'] = [
         {'script':'hg',
          'copyright':'Copyright (C) 2005-2010 Matt Mackall and others',
          'product_version':version}]
 
 if os.name == 'nt':
     # Windows binary file versions for exe/dll files must have the
     # form W.X.Y.Z, where W,X,Y,Z are numbers in the range 0..65535
     setupversion = version.split('+', 1)[0]
 
 if sys.platform == 'darwin' and os.path.exists('/usr/bin/xcodebuild'):
     # XCode 4.0 dropped support for ppc architecture, which is hardcoded in
     # distutils.sysconfig
     version = runcmd(['/usr/bin/xcodebuild', '-version'], {})[0].splitlines()[0]
     # Also parse only first digit, because 3.2.1 can't be parsed nicely
     if (version.startswith('Xcode') and
         StrictVersion(version.split()[1]) >= StrictVersion('4.0')):
         os.environ['ARCHFLAGS'] = '-arch i386 -arch x86_64'
 
 setup(name='mercurial',
       version=setupversion,
       author='Matt Mackall',
       author_email='[e-mail]',
       url='http://mercurial.selenic.com/',
       description='Scalable distributed SCM',
       license='GNU GPLv2+',
       scripts=scripts,
       packages=packages,
       py_modules=pymodules,
       ext_modules=extmodules,
       data_files=datafiles,
       package_data=packagedata,
       cmdclass=cmdclass,
       options=dict(py2exe=dict(packages=['hgext', 'email']),
                    bdist_mpkg=dict(zipdist=True,
                                    license='COPYING',
                                    readme='contrib/macosx/Readme.html',
                                    welcome='contrib/macosx/Welcome.html')),
       **extra)