cleanup: use urllibcompat for renamed methods on urllib request objects

Differential Revision: https://phab.mercurial-scm.org/D891

mercurial/byterange.py

@@ -1,471 +1,472 b''
 #   This library is free software; you can redistribute it and/or
 #   modify it under the terms of the GNU Lesser General Public
 #   License as published by the Free Software Foundation; either
 #   version 2.1 of the License, or (at your option) any later version.
 #
 #   This library is distributed in the hope that it will be useful,
 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 #   Lesser General Public License for more details.
 #
 #   You should have received a copy of the GNU Lesser General Public
 #   License along with this library; if not, see
 #   <http://www.gnu.org/licenses/>.
 
 # This file is part of urlgrabber, a high-level cross-protocol url-grabber
 # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
 
 # $Id: byterange.py,v 1.9 2005/02/14 21:55:07 mstenner Exp $
 
 from __future__ import absolute_import
 
 import email
 import ftplib
 import mimetypes
 import os
 import re
 import socket
 import stat
 
 from . import (
+    urllibcompat,
     util,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 addclosehook = urlreq.addclosehook
 addinfourl = urlreq.addinfourl
 splitattr = urlreq.splitattr
 splitpasswd = urlreq.splitpasswd
 splitport = urlreq.splitport
 splituser = urlreq.splituser
 unquote = urlreq.unquote
 
 class RangeError(IOError):
     """Error raised when an unsatisfiable range is requested."""
 
 class HTTPRangeHandler(urlreq.basehandler):
     """Handler that enables HTTP Range headers.
 
     This was extremely simple. The Range header is a HTTP feature to
     begin with so all this class does is tell urllib2 that the
     "206 Partial Content" response from the HTTP server is what we
     expected.
 
     Example:
         import urllib2
         import byterange
 
         range_handler = range.HTTPRangeHandler()
         opener = urlreq.buildopener(range_handler)
 
         # install it
         urlreq.installopener(opener)
 
         # create Request and set Range header
         req = urlreq.request('http://www.python.org/')
         req.header['Range'] = 'bytes=30-50'
         f = urlreq.urlopen(req)
     """
 
     def http_error_206(self, req, fp, code, msg, hdrs):
         # 206 Partial Content Response
         r = urlreq.addinfourl(fp, hdrs, req.get_full_url())
         r.code = code
         r.msg = msg
         return r
 
     def http_error_416(self, req, fp, code, msg, hdrs):
         # HTTP's Range Not Satisfiable error
         raise RangeError('Requested Range Not Satisfiable')
 
 class RangeableFileObject(object):
     """File object wrapper to enable raw range handling.
     This was implemented primarily for handling range
     specifications for file:// urls. This object effectively makes
     a file object look like it consists only of a range of bytes in
     the stream.
 
     Examples:
         # expose 10 bytes, starting at byte position 20, from
         # /etc/aliases.
         >>> fo = RangeableFileObject(file(b'/etc/passwd', b'r'), (20,30))
         # seek seeks within the range (to position 23 in this case)
         >>> fo.seek(3)
         # tell tells where your at _within the range_ (position 3 in
         # this case)
         >>> fo.tell()
         # read EOFs if an attempt is made to read past the last
         # byte in the range. the following will return only 7 bytes.
         >>> fo.read(30)
     """
 
     def __init__(self, fo, rangetup):
         """Create a RangeableFileObject.
         fo       -- a file like object. only the read() method need be
                     supported but supporting an optimized seek() is
                     preferable.
         rangetup -- a (firstbyte,lastbyte) tuple specifying the range
                     to work over.
         The file object provided is assumed to be at byte offset 0.
         """
         self.fo = fo
         (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup)
         self.realpos = 0
         self._do_seek(self.firstbyte)
 
     def __getattr__(self, name):
         """This effectively allows us to wrap at the instance level.
         Any attribute not found in _this_ object will be searched for
         in self.fo.  This includes methods."""
         return getattr(self.fo, name)
 
     def tell(self):
         """Return the position within the range.
         This is different from fo.seek in that position 0 is the
         first byte position of the range tuple. For example, if
         this object was created with a range tuple of (500,899),
         tell() will return 0 when at byte position 500 of the file.
         """
         return (self.realpos - self.firstbyte)
 
     def seek(self, offset, whence=0):
         """Seek within the byte range.
         Positioning is identical to that described under tell().
         """
         assert whence in (0, 1, 2)
         if whence == 0:   # absolute seek
             realoffset = self.firstbyte + offset
         elif whence == 1: # relative seek
             realoffset = self.realpos + offset
         elif whence == 2: # absolute from end of file
             # XXX: are we raising the right Error here?
             raise IOError('seek from end of file not supported.')
 
         # do not allow seek past lastbyte in range
         if self.lastbyte and (realoffset >= self.lastbyte):
             realoffset = self.lastbyte
 
         self._do_seek(realoffset - self.realpos)
 
     def read(self, size=-1):
         """Read within the range.
         This method will limit the size read based on the range.
         """
         size = self._calc_read_size(size)
         rslt = self.fo.read(size)
         self.realpos += len(rslt)
         return rslt
 
     def readline(self, size=-1):
         """Read lines within the range.
         This method will limit the size read based on the range.
         """
         size = self._calc_read_size(size)
         rslt = self.fo.readline(size)
         self.realpos += len(rslt)
         return rslt
 
     def _calc_read_size(self, size):
         """Handles calculating the amount of data to read based on
         the range.
         """
         if self.lastbyte:
             if size > -1:
                 if ((self.realpos + size) >= self.lastbyte):
                     size = (self.lastbyte - self.realpos)
             else:
                 size = (self.lastbyte - self.realpos)
         return size
 
     def _do_seek(self, offset):
         """Seek based on whether wrapped object supports seek().
         offset is relative to the current position (self.realpos).
         """
         assert offset >= 0
         seek = getattr(self.fo, 'seek', self._poor_mans_seek)
         seek(self.realpos + offset)
         self.realpos += offset
 
     def _poor_mans_seek(self, offset):
         """Seek by calling the wrapped file objects read() method.
         This is used for file like objects that do not have native
         seek support. The wrapped objects read() method is called
         to manually seek to the desired position.
         offset -- read this number of bytes from the wrapped
                   file object.
         raise RangeError if we encounter EOF before reaching the
         specified offset.
         """
         pos = 0
         bufsize = 1024
         while pos < offset:
             if (pos + bufsize) > offset:
                 bufsize = offset - pos
             buf = self.fo.read(bufsize)
             if len(buf) != bufsize:
                 raise RangeError('Requested Range Not Satisfiable')
             pos += bufsize
 
 class FileRangeHandler(urlreq.filehandler):
     """FileHandler subclass that adds Range support.
     This class handles Range headers exactly like an HTTP
     server would.
     """
     def open_local_file(self, req):
-        host = req.get_host()
-        file = req.get_selector()
+        host = urllibcompat.gethost(req)
+        file = urllibcompat.getselector(req)
         localfile = urlreq.url2pathname(file)
         stats = os.stat(localfile)
         size = stats[stat.ST_SIZE]
         modified = email.Utils.formatdate(stats[stat.ST_MTIME])
         mtype = mimetypes.guess_type(file)[0]
         if host:
             host, port = urlreq.splitport(host)
             if port or socket.gethostbyname(host) not in self.get_names():
                 raise urlerr.urlerror('file not on local host')
         fo = open(localfile,'rb')
         brange = req.headers.get('Range', None)
         brange = range_header_to_tuple(brange)
         assert brange != ()
         if brange:
             (fb, lb) = brange
             if lb == '':
                 lb = size
             if fb < 0 or fb > size or lb > size:
                 raise RangeError('Requested Range Not Satisfiable')
             size = (lb - fb)
             fo = RangeableFileObject(fo, (fb, lb))
         headers = email.message_from_string(
             'Content-Type: %s\nContent-Length: %d\nLast-Modified: %s\n' %
             (mtype or 'text/plain', size, modified))
         return urlreq.addinfourl(fo, headers, 'file:'+file)
 
 
 # FTP Range Support
 # Unfortunately, a large amount of base FTP code had to be copied
 # from urllib and urllib2 in order to insert the FTP REST command.
 # Code modifications for range support have been commented as
 # follows:
 # -- range support modifications start/end here
 
 class FTPRangeHandler(urlreq.ftphandler):
     def ftp_open(self, req):
-        host = req.get_host()
+        host = urllibcompat.gethost(req)
         if not host:
             raise IOError('ftp error', 'no host given')
         host, port = splitport(host)
         if port is None:
             port = ftplib.FTP_PORT
         else:
             port = int(port)
 
         # username/password handling
         user, host = splituser(host)
         if user:
             user, passwd = splitpasswd(user)
         else:
             passwd = None
         host = unquote(host)
         user = unquote(user or '')
         passwd = unquote(passwd or '')
 
         try:
             host = socket.gethostbyname(host)
         except socket.error as msg:
             raise urlerr.urlerror(msg)
         path, attrs = splitattr(req.get_selector())
         dirs = path.split('/')
         dirs = map(unquote, dirs)
         dirs, file = dirs[:-1], dirs[-1]
         if dirs and not dirs[0]:
             dirs = dirs[1:]
         try:
             fw = self.connect_ftp(user, passwd, host, port, dirs)
             if file:
                 type = 'I'
             else:
                 type = 'D'
 
             for attr in attrs:
                 attr, value = splitattr(attr)
                 if attr.lower() == 'type' and \
                    value in ('a', 'A', 'i', 'I', 'd', 'D'):
                     type = value.upper()
 
             # -- range support modifications start here
             rest = None
             range_tup = range_header_to_tuple(req.headers.get('Range', None))
             assert range_tup != ()
             if range_tup:
                 (fb, lb) = range_tup
                 if fb > 0:
                     rest = fb
             # -- range support modifications end here
 
             fp, retrlen = fw.retrfile(file, type, rest)
 
             # -- range support modifications start here
             if range_tup:
                 (fb, lb) = range_tup
                 if lb == '':
                     if retrlen is None or retrlen == 0:
                         raise RangeError('Requested Range Not Satisfiable due'
                                          ' to unobtainable file length.')
                     lb = retrlen
                     retrlen = lb - fb
                     if retrlen < 0:
                         # beginning of range is larger than file
                         raise RangeError('Requested Range Not Satisfiable')
                 else:
                     retrlen = lb - fb
                     fp = RangeableFileObject(fp, (0, retrlen))
             # -- range support modifications end here
 
             headers = ""
             mtype = mimetypes.guess_type(req.get_full_url())[0]
             if mtype:
                 headers += "Content-Type: %s\n" % mtype
             if retrlen is not None and retrlen >= 0:
                 headers += "Content-Length: %d\n" % retrlen
             headers = email.message_from_string(headers)
             return addinfourl(fp, headers, req.get_full_url())
         except ftplib.all_errors as msg:
             raise IOError('ftp error', msg)
 
     def connect_ftp(self, user, passwd, host, port, dirs):
         fw = ftpwrapper(user, passwd, host, port, dirs)
         return fw
 
 class ftpwrapper(urlreq.ftpwrapper):
     # range support note:
     # this ftpwrapper code is copied directly from
     # urllib. The only enhancement is to add the rest
     # argument and pass it on to ftp.ntransfercmd
     def retrfile(self, file, type, rest=None):
         self.endtransfer()
         if type in ('d', 'D'):
             cmd = 'TYPE A'
             isdir = 1
         else:
             cmd = 'TYPE ' + type
             isdir = 0
         try:
             self.ftp.voidcmd(cmd)
         except ftplib.all_errors:
             self.init()
             self.ftp.voidcmd(cmd)
         conn = None
         if file and not isdir:
             # Use nlst to see if the file exists at all
             try:
                 self.ftp.nlst(file)
             except ftplib.error_perm as reason:
                 raise IOError('ftp error', reason)
             # Restore the transfer mode!
             self.ftp.voidcmd(cmd)
             # Try to retrieve as a file
             try:
                 cmd = 'RETR ' + file
                 conn = self.ftp.ntransfercmd(cmd, rest)
             except ftplib.error_perm as reason:
                 if str(reason).startswith('501'):
                     # workaround for REST not supported error
                     fp, retrlen = self.retrfile(file, type)
                     fp = RangeableFileObject(fp, (rest,''))
                     return (fp, retrlen)
                 elif not str(reason).startswith('550'):
                     raise IOError('ftp error', reason)
         if not conn:
             # Set transfer mode to ASCII!
             self.ftp.voidcmd('TYPE A')
             # Try a directory listing
             if file:
                 cmd = 'LIST ' + file
             else:
                 cmd = 'LIST'
             conn = self.ftp.ntransfercmd(cmd)
         self.busy = 1
         # Pass back both a suitably decorated object and a retrieval length
         return (addclosehook(conn[0].makefile('rb'),
                             self.endtransfer), conn[1])
 
 
 ####################################################################
 # Range Tuple Functions
 # XXX: These range tuple functions might go better in a class.
 
 _rangere = None
 def range_header_to_tuple(range_header):
     """Get a (firstbyte,lastbyte) tuple from a Range header value.
 
     Range headers have the form "bytes=<firstbyte>-<lastbyte>". This
     function pulls the firstbyte and lastbyte values and returns
     a (firstbyte,lastbyte) tuple. If lastbyte is not specified in
     the header value, it is returned as an empty string in the
     tuple.
 
     Return None if range_header is None
     Return () if range_header does not conform to the range spec
     pattern.
 
     """
     global _rangere
     if range_header is None:
         return None
     if _rangere is None:
         _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)')
     match = _rangere.match(range_header)
     if match:
         tup = range_tuple_normalize(match.group(1, 2))
         if tup and tup[1]:
             tup = (tup[0], tup[1]+1)
         return tup
     return ()
 
 def range_tuple_to_header(range_tup):
     """Convert a range tuple to a Range header value.
     Return a string of the form "bytes=<firstbyte>-<lastbyte>" or None
     if no range is needed.
     """
     if range_tup is None:
         return None
     range_tup = range_tuple_normalize(range_tup)
     if range_tup:
         if range_tup[1]:
             range_tup = (range_tup[0], range_tup[1] - 1)
         return 'bytes=%s-%s' % range_tup
 
 def range_tuple_normalize(range_tup):
     """Normalize a (first_byte,last_byte) range tuple.
     Return a tuple whose first element is guaranteed to be an int
     and whose second element will be '' (meaning: the last byte) or
     an int. Finally, return None if the normalized tuple == (0,'')
     as that is equivalent to retrieving the entire file.
     """
     if range_tup is None:
         return None
     # handle first byte
     fb = range_tup[0]
     if fb in (None, ''):
         fb = 0
     else:
         fb = int(fb)
     # handle last byte
     try:
         lb = range_tup[1]
     except IndexError:
         lb = ''
     else:
         if lb is None:
             lb = ''
         elif lb != '':
             lb = int(lb)
     # check if range is over the entire file
     if (fb, lb) == (0, ''):
         return None
     # check that the range is valid
     if lb < fb:
         raise RangeError('Invalid byte range: %s-%s' % (fb, lb))
     return (fb, lb)

mercurial/httpconnection.py

@@ -1,297 +1,299 b''
 # httpconnection.py - urllib2 handler for new http support
 #
 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <[e-mail]>
 # Copyright 2006, 2007 Alexis S. L. Carvalho <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 # Copyright 2011 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import logging
 import os
 import socket
 
 from .i18n import _
 from . import (
     httpclient,
     sslutil,
+    urllibcompat,
     util,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # moved here from url.py to avoid a cycle
 class httpsendfile(object):
     """This is a wrapper around the objects returned by python's "open".
 
     Its purpose is to send file-like objects via HTTP.
     It do however not define a __len__ attribute because the length
     might be more than Py_ssize_t can handle.
     """
 
     def __init__(self, ui, *args, **kwargs):
         self.ui = ui
         self._data = open(*args, **kwargs)
         self.seek = self._data.seek
         self.close = self._data.close
         self.write = self._data.write
         self.length = os.fstat(self._data.fileno()).st_size
         self._pos = 0
         self._total = self.length // 1024 * 2
 
     def read(self, *args, **kwargs):
         ret = self._data.read(*args, **kwargs)
         if not ret:
             self.ui.progress(_('sending'), None)
             return ret
         self._pos += len(ret)
         # We pass double the max for total because we currently have
         # to send the bundle twice in the case of a server that
         # requires authentication. Since we can't know until we try
         # once whether authentication will be required, just lie to
         # the user and maybe the push succeeds suddenly at 50%.
         self.ui.progress(_('sending'), self._pos // 1024,
                          unit=_('kb'), total=self._total)
         return ret
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
 # moved here from url.py to avoid a cycle
 def readauthforuri(ui, uri, user):
     # Read configuration
     groups = {}
     for key, val in ui.configitems('auth'):
         if key in ('cookiefile',):
             continue
 
         if '.' not in key:
             ui.warn(_("ignoring invalid [auth] key '%s'\n") % key)
             continue
         group, setting = key.rsplit('.', 1)
         gdict = groups.setdefault(group, {})
         if setting in ('username', 'cert', 'key'):
             val = util.expandpath(val)
         gdict[setting] = val
 
     # Find the best match
     scheme, hostpath = uri.split('://', 1)
     bestuser = None
     bestlen = 0
     bestauth = None
     for group, auth in groups.iteritems():
         if user and user != auth.get('username', user):
             # If a username was set in the URI, the entry username
             # must either match it or be unset
             continue
         prefix = auth.get('prefix')
         if not prefix:
             continue
         p = prefix.split('://', 1)
         if len(p) > 1:
             schemes, prefix = [p[0]], p[1]
         else:
             schemes = (auth.get('schemes') or 'https').split()
         if (prefix == '*' or hostpath.startswith(prefix)) and \
             (len(prefix) > bestlen or (len(prefix) == bestlen and \
                 not bestuser and 'username' in auth)) \
              and scheme in schemes:
             bestlen = len(prefix)
             bestauth = group, auth
             bestuser = auth.get('username')
             if user and not bestuser:
                 auth['username'] = user
     return bestauth
 
 # Mercurial (at least until we can remove the old codepath) requires
 # that the http response object be sufficiently file-like, so we
 # provide a close() method here.
 class HTTPResponse(httpclient.HTTPResponse):
     def close(self):
         pass
 
 class HTTPConnection(httpclient.HTTPConnection):
     response_class = HTTPResponse
     def request(self, method, uri, body=None, headers=None):
         if headers is None:
             headers = {}
         if isinstance(body, httpsendfile):
             body.seek(0)
         httpclient.HTTPConnection.request(self, method, uri, body=body,
                                           headers=headers)
 
 
 _configuredlogging = False
 LOGFMT = '%(levelname)s:%(name)s:%(lineno)d:%(message)s'
 # Subclass BOTH of these because otherwise urllib2 "helpfully"
 # reinserts them since it notices we don't include any subclasses of
 # them.
 class http2handler(urlreq.httphandler, urlreq.httpshandler):
     def __init__(self, ui, pwmgr):
         global _configuredlogging
         urlreq.abstracthttphandler.__init__(self)
         self.ui = ui
         self.pwmgr = pwmgr
         self._connections = {}
         # developer config: ui.http2debuglevel
         loglevel = ui.config('ui', 'http2debuglevel')
         if loglevel and not _configuredlogging:
             _configuredlogging = True
             logger = logging.getLogger('mercurial.httpclient')
             logger.setLevel(getattr(logging, loglevel.upper()))
             handler = logging.StreamHandler()
             handler.setFormatter(logging.Formatter(LOGFMT))
             logger.addHandler(handler)
 
     def close_all(self):
         """Close and remove all connection objects being kept for reuse."""
         for openconns in self._connections.values():
             for conn in openconns:
                 conn.close()
         self._connections = {}
 
     # shamelessly borrowed from urllib2.AbstractHTTPHandler
     def do_open(self, http_class, req, use_ssl):
         """Return an addinfourl object for the request, using http_class.
 
         http_class must implement the HTTPConnection API from httplib.
         The addinfourl return value is a file-like object.  It also
         has methods and attributes including:
             - info(): return a mimetools.Message object for the headers
             - geturl(): return the original request URL
             - code: HTTP status code
         """
         # If using a proxy, the host returned by get_host() is
         # actually the proxy. On Python 2.6.1, the real destination
         # hostname is encoded in the URI in the urllib2 request
         # object. On Python 2.6.5, it's stored in the _tunnel_host
         # attribute which has no accessor.
         tunhost = getattr(req, '_tunnel_host', None)
-        host = req.get_host()
+        host = urllibcompat.gethost(req)
         if tunhost:
             proxyhost = host
             host = tunhost
         elif req.has_proxy():
-            proxyhost = req.get_host()
-            host = req.get_selector().split('://', 1)[1].split('/', 1)[0]
+            proxyhost = urllibcompat.gethost(req)
+            host = urllibcompat.getselector(
+                req).split('://', 1)[1].split('/', 1)[0]
         else:
             proxyhost = None
 
         if proxyhost:
             if ':' in proxyhost:
                 # Note: this means we'll explode if we try and use an
                 # IPv6 http proxy. This isn't a regression, so we
                 # won't worry about it for now.
                 proxyhost, proxyport = proxyhost.rsplit(':', 1)
             else:
                 proxyport = 3128 # squid default
             proxy = (proxyhost, proxyport)
         else:
             proxy = None
 
         if not host:
             raise urlerr.urlerror('no host given')
 
         connkey = use_ssl, host, proxy
         allconns = self._connections.get(connkey, [])
         conns = [c for c in allconns if not c.busy()]
         if conns:
             h = conns[0]
         else:
             if allconns:
                 self.ui.debug('all connections for %s busy, making a new '
                               'one\n' % host)
             timeout = None
             if req.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
                 timeout = req.timeout
             h = http_class(host, timeout=timeout, proxy_hostport=proxy)
             self._connections.setdefault(connkey, []).append(h)
 
         headers = dict(req.headers)
         headers.update(req.unredirected_hdrs)
         headers = dict(
             (name.title(), val) for name, val in headers.items())
         try:
-            path = req.get_selector()
+            path = urllibcompat.getselector(req)
             if '://' in path:
                 path = path.split('://', 1)[1].split('/', 1)[1]
             if path[0] != '/':
                 path = '/' + path
             h.request(req.get_method(), path, req.data, headers)
             r = h.getresponse()
         except socket.error as err: # XXX what error?
             raise urlerr.urlerror(err)
 
         # Pick apart the HTTPResponse object to get the addinfourl
         # object initialized properly.
         r.recv = r.read
 
-        resp = urlreq.addinfourl(r, r.headers, req.get_full_url())
+        resp = urlreq.addinfourl(r, r.headers, urllibcompat.getfullurl(req))
         resp.code = r.status
         resp.msg = r.reason
         return resp
 
     # httplib always uses the given host/port as the socket connect
     # target, and then allows full URIs in the request path, which it
     # then observes and treats as a signal to do proxying instead.
     def http_open(self, req):
-        if req.get_full_url().startswith('https'):
+        if urllibcompat.getfullurl(req).startswith('https'):
             return self.https_open(req)
         def makehttpcon(*args, **kwargs):
             k2 = dict(kwargs)
             k2['use_ssl'] = False
             return HTTPConnection(*args, **k2)
         return self.do_open(makehttpcon, req, False)
 
     def https_open(self, req):
-        # req.get_full_url() does not contain credentials and we may
+        # urllibcompat.getfullurl(req) does not contain credentials and we may
         # need them to match the certificates.
-        url = req.get_full_url()
+        url = urllibcompat.getfullurl(req)
         user, password = self.pwmgr.find_stored_password(url)
         res = readauthforuri(self.ui, url, user)
         if res:
             group, auth = res
             self.auth = auth
             self.ui.debug("using auth.%s.* for authentication\n" % group)
         else:
             self.auth = None
         return self.do_open(self._makesslconnection, req, True)
 
     def _makesslconnection(self, host, port=443, *args, **kwargs):
         keyfile = None
         certfile = None
 
         if args: # key_file
             keyfile = args.pop(0)
         if args: # cert_file
             certfile = args.pop(0)
 
         # if the user has specified different key/cert files in
         # hgrc, we prefer these
         if self.auth and 'key' in self.auth and 'cert' in self.auth:
             keyfile = self.auth['key']
             certfile = self.auth['cert']
 
         # let host port take precedence
         if ':' in host and '[' not in host or ']:' in host:
             host, port = host.rsplit(':', 1)
             port = int(port)
             if '[' in host:
                 host = host[1:-1]
 
         kwargs['keyfile'] = keyfile
         kwargs['certfile'] = certfile
 
         con = HTTPConnection(host, port, use_ssl=True,
                              ssl_wrap_socket=sslutil.wrapsocket,
                              ssl_validator=sslutil.validatesocket,
                              ui=self.ui,
                              **kwargs)
         return con

mercurial/keepalive.py

@@ -1,723 +1,726 b''
 #   This library is free software; you can redistribute it and/or
 #   modify it under the terms of the GNU Lesser General Public
 #   License as published by the Free Software Foundation; either
 #   version 2.1 of the License, or (at your option) any later version.
 #
 #   This library is distributed in the hope that it will be useful,
 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 #   Lesser General Public License for more details.
 #
 #   You should have received a copy of the GNU Lesser General Public
 #   License along with this library; if not, see
 #   <http://www.gnu.org/licenses/>.
 
 # This file is part of urlgrabber, a high-level cross-protocol url-grabber
 # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
 
 # Modified by Benoit Boissinot:
 #  - fix for digest auth (inspired from urllib2.py @ Python v2.4)
 # Modified by Dirkjan Ochtman:
 #  - import md5 function from a local util module
 # Modified by Augie Fackler:
 #  - add safesend method and use it to prevent broken pipe errors
 #    on large POST requests
 
 """An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
 
 >>> import urllib2
 >>> from keepalive import HTTPHandler
 >>> keepalive_handler = HTTPHandler()
 >>> opener = urlreq.buildopener(keepalive_handler)
 >>> urlreq.installopener(opener)
 >>>
 >>> fo = urlreq.urlopen('http://www.python.org')
 
 If a connection to a given host is requested, and all of the existing
 connections are still in use, another connection will be opened.  If
 the handler tries to use an existing connection but it fails in some
 way, it will be closed and removed from the pool.
 
 To remove the handler, simply re-run build_opener with no arguments, and
 install that opener.
 
 You can explicitly close connections by using the close_connection()
 method of the returned file-like object (described below) or you can
 use the handler methods:
 
   close_connection(host)
   close_all()
   open_connections()
 
 NOTE: using the close_connection and close_all methods of the handler
 should be done with care when using multiple threads.
   * there is nothing that prevents another thread from creating new
     connections immediately after connections are closed
   * no checks are done to prevent in-use connections from being closed
 
 >>> keepalive_handler.close_all()
 
 EXTRA ATTRIBUTES AND METHODS
 
   Upon a status of 200, the object returned has a few additional
   attributes and methods, which should not be used if you want to
   remain consistent with the normal urllib2-returned objects:
 
     close_connection()  -  close the connection to the host
     readlines()         -  you know, readlines()
     status              -  the return status (i.e. 404)
     reason              -  english translation of status (i.e. 'File not found')
 
   If you want the best of both worlds, use this inside an
   AttributeError-catching try:
 
   >>> try: status = fo.status
   >>> except AttributeError: status = None
 
   Unfortunately, these are ONLY there if status == 200, so it's not
   easy to distinguish between non-200 responses.  The reason is that
   urllib2 tries to do clever things with error codes 301, 302, 401,
   and 407, and it wraps the object upon return.
 """
 
 # $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $
 
 from __future__ import absolute_import, print_function
 
 import errno
 import hashlib
 import socket
 import sys
 import threading
 
 from .i18n import _
 from . import (
     pycompat,
+    urllibcompat,
     util,
 )
 
 httplib = util.httplib
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 DEBUG = None
 
 class ConnectionManager(object):
     """
     The connection manager must be able to:
       * keep track of all existing
       """
     def __init__(self):
         self._lock = threading.Lock()
         self._hostmap = {} # map hosts to a list of connections
         self._connmap = {} # map connections to host
         self._readymap = {} # map connection to ready state
 
     def add(self, host, connection, ready):
         self._lock.acquire()
         try:
             if host not in self._hostmap:
                 self._hostmap[host] = []
             self._hostmap[host].append(connection)
             self._connmap[connection] = host
             self._readymap[connection] = ready
         finally:
             self._lock.release()
 
     def remove(self, connection):
         self._lock.acquire()
         try:
             try:
                 host = self._connmap[connection]
             except KeyError:
                 pass
             else:
                 del self._connmap[connection]
                 del self._readymap[connection]
                 self._hostmap[host].remove(connection)
                 if not self._hostmap[host]:
                     del self._hostmap[host]
         finally:
             self._lock.release()
 
     def set_ready(self, connection, ready):
         try:
             self._readymap[connection] = ready
         except KeyError:
             pass
 
     def get_ready_conn(self, host):
         conn = None
         self._lock.acquire()
         try:
             if host in self._hostmap:
                 for c in self._hostmap[host]:
                     if self._readymap[c]:
                         self._readymap[c] = 0
                         conn = c
                         break
         finally:
             self._lock.release()
         return conn
 
     def get_all(self, host=None):
         if host:
             return list(self._hostmap.get(host, []))
         else:
             return dict(self._hostmap)
 
 class KeepAliveHandler(object):
     def __init__(self):
         self._cm = ConnectionManager()
 
     #### Connection Management
     def open_connections(self):
         """return a list of connected hosts and the number of connections
         to each.  [('foo.com:80', 2), ('bar.org', 1)]"""
         return [(host, len(li)) for (host, li) in self._cm.get_all().items()]
 
     def close_connection(self, host):
         """close connection(s) to <host>
         host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
         no error occurs if there is no connection to that host."""
         for h in self._cm.get_all(host):
             self._cm.remove(h)
             h.close()
 
     def close_all(self):
         """close all open connections"""
         for host, conns in self._cm.get_all().iteritems():
             for h in conns:
                 self._cm.remove(h)
                 h.close()
 
     def _request_closed(self, request, host, connection):
         """tells us that this request is now closed and that the
         connection is ready for another request"""
         self._cm.set_ready(connection, 1)
 
     def _remove_connection(self, host, connection, close=0):
         if close:
             connection.close()
         self._cm.remove(connection)
 
     #### Transaction Execution
     def http_open(self, req):
         return self.do_open(HTTPConnection, req)
 
     def do_open(self, http_class, req):
-        host = req.get_host()
+        host = urllibcompat.gethost(req)
         if not host:
             raise urlerr.urlerror('no host given')
 
         try:
             h = self._cm.get_ready_conn(host)
             while h:
                 r = self._reuse_connection(h, req, host)
 
                 # if this response is non-None, then it worked and we're
                 # done.  Break out, skipping the else block.
                 if r:
                     break
 
                 # connection is bad - possibly closed by server
                 # discard it and ask for the next free connection
                 h.close()
                 self._cm.remove(h)
                 h = self._cm.get_ready_conn(host)
             else:
                 # no (working) free connections were found.  Create a new one.
                 h = http_class(host)
                 if DEBUG:
                     DEBUG.info("creating new connection to %s (%d)",
                                host, id(h))
                 self._cm.add(host, h, 0)
                 self._start_transaction(h, req)
                 r = h.getresponse()
         # The string form of BadStatusLine is the status line. Add some context
         # to make the error message slightly more useful.
         except httplib.BadStatusLine as err:
             raise urlerr.urlerror(
                 _('bad HTTP status line: %s') % pycompat.sysbytes(err.line))
         except (socket.error, httplib.HTTPException) as err:
             raise urlerr.urlerror(err)
 
         # if not a persistent connection, don't try to reuse it
         if r.will_close:
             self._cm.remove(h)
 
         if DEBUG:
             DEBUG.info("STATUS: %s, %s", r.status, r.reason)
         r._handler = self
         r._host = host
         r._url = req.get_full_url()
         r._connection = h
         r.code = r.status
         r.headers = r.msg
         r.msg = r.reason
 
         return r
 
     def _reuse_connection(self, h, req, host):
         """start the transaction with a re-used connection
         return a response object (r) upon success or None on failure.
         This DOES not close or remove bad connections in cases where
         it returns.  However, if an unexpected exception occurs, it
         will close and remove the connection before re-raising.
         """
         try:
             self._start_transaction(h, req)
             r = h.getresponse()
             # note: just because we got something back doesn't mean it
             # worked.  We'll check the version below, too.
         except (socket.error, httplib.HTTPException):
             r = None
         except: # re-raises
             # adding this block just in case we've missed
             # something we will still raise the exception, but
             # lets try and close the connection and remove it
             # first.  We previously got into a nasty loop
             # where an exception was uncaught, and so the
             # connection stayed open.  On the next try, the
             # same exception was raised, etc.  The trade-off is
             # that it's now possible this call will raise
             # a DIFFERENT exception
             if DEBUG:
                 DEBUG.error("unexpected exception - closing "
                             "connection to %s (%d)", host, id(h))
             self._cm.remove(h)
             h.close()
             raise
 
         if r is None or r.version == 9:
             # httplib falls back to assuming HTTP 0.9 if it gets a
             # bad header back.  This is most likely to happen if
             # the socket has been closed by the server since we
             # last used the connection.
             if DEBUG:
                 DEBUG.info("failed to re-use connection to %s (%d)",
                            host, id(h))
             r = None
         else:
             if DEBUG:
                 DEBUG.info("re-using connection to %s (%d)", host, id(h))
 
         return r
 
     def _start_transaction(self, h, req):
         # What follows mostly reimplements HTTPConnection.request()
         # except it adds self.parent.addheaders in the mix and sends headers
         # in a deterministic order (to make testing easier).
         headers = util.sortdict(self.parent.addheaders)
         headers.update(sorted(req.headers.items()))
         headers.update(sorted(req.unredirected_hdrs.items()))
         headers = util.sortdict((n.lower(), v) for n, v in headers.items())
         skipheaders = {}
         for n in ('host', 'accept-encoding'):
             if n in headers:
                 skipheaders['skip_' + n.replace('-', '_')] = 1
         try:
-            if req.has_data():
-                data = req.get_data()
+            if urllibcompat.hasdata(req):
+                data = urllibcompat.getdata(req)
                 h.putrequest(
-                    req.get_method(), req.get_selector(), **skipheaders)
+                    req.get_method(), urllibcompat.getselector(req),
+                    **skipheaders)
                 if 'content-type' not in headers:
                     h.putheader('Content-type',
                                 'application/x-www-form-urlencoded')
                 if 'content-length' not in headers:
                     h.putheader('Content-length', '%d' % len(data))
             else:
                 h.putrequest(
-                    req.get_method(), req.get_selector(), **skipheaders)
+                    req.get_method(), urllibcompat.getselector(req),
+                    **skipheaders)
         except socket.error as err:
             raise urlerr.urlerror(err)
         for k, v in headers.items():
             h.putheader(k, v)
         h.endheaders()
-        if req.has_data():
+        if urllibcompat.hasdata(req):
             h.send(data)
 
 class HTTPHandler(KeepAliveHandler, urlreq.httphandler):
     pass
 
 class HTTPResponse(httplib.HTTPResponse):
     # we need to subclass HTTPResponse in order to
     # 1) add readline() and readlines() methods
     # 2) add close_connection() methods
     # 3) add info() and geturl() methods
 
     # in order to add readline(), read must be modified to deal with a
     # buffer.  example: readline must read a buffer and then spit back
     # one line at a time.  The only real alternative is to read one
     # BYTE at a time (ick).  Once something has been read, it can't be
     # put back (ok, maybe it can, but that's even uglier than this),
     # so if you THEN do a normal read, you must first take stuff from
     # the buffer.
 
     # the read method wraps the original to accommodate buffering,
     # although read() never adds to the buffer.
     # Both readline and readlines have been stolen with almost no
     # modification from socket.py
 
 
     def __init__(self, sock, debuglevel=0, strict=0, method=None):
         extrakw = {}
         if not pycompat.ispy3:
             extrakw['strict'] = True
             extrakw['buffering'] = True
         httplib.HTTPResponse.__init__(self, sock, debuglevel=debuglevel,
                                       method=method, **extrakw)
         self.fileno = sock.fileno
         self.code = None
         self._rbuf = ''
         self._rbufsize = 8096
         self._handler = None # inserted by the handler later
         self._host = None    # (same)
         self._url = None     # (same)
         self._connection = None # (same)
 
     _raw_read = httplib.HTTPResponse.read
 
     def close(self):
         if self.fp:
             self.fp.close()
             self.fp = None
             if self._handler:
                 self._handler._request_closed(self, self._host,
                                               self._connection)
 
     def close_connection(self):
         self._handler._remove_connection(self._host, self._connection, close=1)
         self.close()
 
     def info(self):
         return self.headers
 
     def geturl(self):
         return self._url
 
     def read(self, amt=None):
         # the _rbuf test is only in this first if for speed.  It's not
         # logically necessary
         if self._rbuf and amt is not None:
             L = len(self._rbuf)
             if amt > L:
                 amt -= L
             else:
                 s = self._rbuf[:amt]
                 self._rbuf = self._rbuf[amt:]
                 return s
 
         s = self._rbuf + self._raw_read(amt)
         self._rbuf = ''
         return s
 
     # stolen from Python SVN #68532 to fix issue1088
     def _read_chunked(self, amt):
         chunk_left = self.chunk_left
         parts = []
 
         while True:
             if chunk_left is None:
                 line = self.fp.readline()
                 i = line.find(';')
                 if i >= 0:
                     line = line[:i] # strip chunk-extensions
                 try:
                     chunk_left = int(line, 16)
                 except ValueError:
                     # close the connection as protocol synchronization is
                     # probably lost
                     self.close()
                     raise httplib.IncompleteRead(''.join(parts))
                 if chunk_left == 0:
                     break
             if amt is None:
                 parts.append(self._safe_read(chunk_left))
             elif amt < chunk_left:
                 parts.append(self._safe_read(amt))
                 self.chunk_left = chunk_left - amt
                 return ''.join(parts)
             elif amt == chunk_left:
                 parts.append(self._safe_read(amt))
                 self._safe_read(2)  # toss the CRLF at the end of the chunk
                 self.chunk_left = None
                 return ''.join(parts)
             else:
                 parts.append(self._safe_read(chunk_left))
                 amt -= chunk_left
 
             # we read the whole chunk, get another
             self._safe_read(2)      # toss the CRLF at the end of the chunk
             chunk_left = None
 
         # read and discard trailer up to the CRLF terminator
         ### note: we shouldn't have any trailers!
         while True:
             line = self.fp.readline()
             if not line:
                 # a vanishingly small number of sites EOF without
                 # sending the trailer
                 break
             if line == '\r\n':
                 break
 
         # we read everything; close the "file"
         self.close()
 
         return ''.join(parts)
 
     def readline(self):
         # Fast path for a line is already available in read buffer.
         i = self._rbuf.find('\n')
         if i >= 0:
             i += 1
             line = self._rbuf[:i]
             self._rbuf = self._rbuf[i:]
             return line
 
         # No newline in local buffer. Read until we find one.
         chunks = [self._rbuf]
         i = -1
         readsize = self._rbufsize
         while True:
             new = self._raw_read(readsize)
             if not new:
                 break
 
             chunks.append(new)
             i = new.find('\n')
             if i >= 0:
                 break
 
         # We either have exhausted the stream or have a newline in chunks[-1].
 
         # EOF
         if i == -1:
             self._rbuf = ''
             return ''.join(chunks)
 
         i += 1
         self._rbuf = chunks[-1][i:]
         chunks[-1] = chunks[-1][:i]
         return ''.join(chunks)
 
     def readlines(self, sizehint=0):
         total = 0
         list = []
         while True:
             line = self.readline()
             if not line:
                 break
             list.append(line)
             total += len(line)
             if sizehint and total >= sizehint:
                 break
         return list
 
 def safesend(self, str):
     """Send `str' to the server.
 
     Shamelessly ripped off from httplib to patch a bad behavior.
     """
     # _broken_pipe_resp is an attribute we set in this function
     # if the socket is closed while we're sending data but
     # the server sent us a response before hanging up.
     # In that case, we want to pretend to send the rest of the
     # outgoing data, and then let the user use getresponse()
     # (which we wrap) to get this last response before
     # opening a new socket.
     if getattr(self, '_broken_pipe_resp', None) is not None:
         return
 
     if self.sock is None:
         if self.auto_open:
             self.connect()
         else:
             raise httplib.NotConnected
 
     # send the data to the server. if we get a broken pipe, then close
     # the socket. we want to reconnect when somebody tries to send again.
     #
     # NOTE: we DO propagate the error, though, because we cannot simply
     #       ignore the error... the caller will know if they can retry.
     if self.debuglevel > 0:
         print("send:", repr(str))
     try:
         blocksize = 8192
         read = getattr(str, 'read', None)
         if read is not None:
             if self.debuglevel > 0:
                 print("sending a read()able")
             data = read(blocksize)
             while data:
                 self.sock.sendall(data)
                 data = read(blocksize)
         else:
             self.sock.sendall(str)
     except socket.error as v:
         reraise = True
         if v[0] == errno.EPIPE:      # Broken pipe
             if self._HTTPConnection__state == httplib._CS_REQ_SENT:
                 self._broken_pipe_resp = None
                 self._broken_pipe_resp = self.getresponse()
                 reraise = False
             self.close()
         if reraise:
             raise
 
 def wrapgetresponse(cls):
     """Wraps getresponse in cls with a broken-pipe sane version.
     """
     def safegetresponse(self):
         # In safesend() we might set the _broken_pipe_resp
         # attribute, in which case the socket has already
         # been closed and we just need to give them the response
         # back. Otherwise, we use the normal response path.
         r = getattr(self, '_broken_pipe_resp', None)
         if r is not None:
             return r
         return cls.getresponse(self)
     safegetresponse.__doc__ = cls.getresponse.__doc__
     return safegetresponse
 
 class HTTPConnection(httplib.HTTPConnection):
     # use the modified response class
     response_class = HTTPResponse
     send = safesend
     getresponse = wrapgetresponse(httplib.HTTPConnection)
 
 
 #########################################################################
 #####   TEST FUNCTIONS
 #########################################################################
 
 
 def continuity(url):
     md5 = hashlib.md5
     format = '%25s: %s'
 
     # first fetch the file with the normal http handler
     opener = urlreq.buildopener()
     urlreq.installopener(opener)
     fo = urlreq.urlopen(url)
     foo = fo.read()
     fo.close()
     m = md5(foo)
     print(format % ('normal urllib', m.hexdigest()))
 
     # now install the keepalive handler and try again
     opener = urlreq.buildopener(HTTPHandler())
     urlreq.installopener(opener)
 
     fo = urlreq.urlopen(url)
     foo = fo.read()
     fo.close()
     m = md5(foo)
     print(format % ('keepalive read', m.hexdigest()))
 
     fo = urlreq.urlopen(url)
     foo = ''
     while True:
         f = fo.readline()
         if f:
             foo = foo + f
         else:
             break
     fo.close()
     m = md5(foo)
     print(format % ('keepalive readline', m.hexdigest()))
 
 def comp(N, url):
     print('  making %i connections to:\n  %s' % (N, url))
 
     util.stdout.write('  first using the normal urllib handlers')
     # first use normal opener
     opener = urlreq.buildopener()
     urlreq.installopener(opener)
     t1 = fetch(N, url)
     print('  TIME: %.3f s' % t1)
 
     util.stdout.write('  now using the keepalive handler       ')
     # now install the keepalive handler and try again
     opener = urlreq.buildopener(HTTPHandler())
     urlreq.installopener(opener)
     t2 = fetch(N, url)
     print('  TIME: %.3f s' % t2)
     print('  improvement factor: %.2f' % (t1 / t2))
 
 def fetch(N, url, delay=0):
     import time
     lens = []
     starttime = time.time()
     for i in range(N):
         if delay and i > 0:
             time.sleep(delay)
         fo = urlreq.urlopen(url)
         foo = fo.read()
         fo.close()
         lens.append(len(foo))
     diff = time.time() - starttime
 
     j = 0
     for i in lens[1:]:
         j = j + 1
         if not i == lens[0]:
             print("WARNING: inconsistent length on read %i: %i" % (j, i))
 
     return diff
 
 def test_timeout(url):
     global DEBUG
     dbbackup = DEBUG
     class FakeLogger(object):
         def debug(self, msg, *args):
             print(msg % args)
         info = warning = error = debug
     DEBUG = FakeLogger()
     print("  fetching the file to establish a connection")
     fo = urlreq.urlopen(url)
     data1 = fo.read()
     fo.close()
 
     i = 20
     print("  waiting %i seconds for the server to close the connection" % i)
     while i > 0:
         util.stdout.write('\r  %2i' % i)
         util.stdout.flush()
         time.sleep(1)
         i -= 1
     util.stderr.write('\r')
 
     print("  fetching the file a second time")
     fo = urlreq.urlopen(url)
     data2 = fo.read()
     fo.close()
 
     if data1 == data2:
         print('  data are identical')
     else:
         print('  ERROR: DATA DIFFER')
 
     DEBUG = dbbackup
 
 
 def test(url, N=10):
     print("performing continuity test (making sure stuff isn't corrupted)")
     continuity(url)
     print('')
     print("performing speed comparison")
     comp(N, url)
     print('')
     print("performing dropped-connection check")
     test_timeout(url)
 
 if __name__ == '__main__':
     import time
     try:
         N = int(sys.argv[1])
         url = sys.argv[2]
     except (IndexError, ValueError):
         print("%s <integer> <url>" % sys.argv[0])
     else:
         test(url, N)

mercurial/url.py

@@ -1,517 +1,519 b''
 # url.py - HTTP handling for mercurial
 #
 # Copyright 2005, 2006, 2007, 2008 Matt Mackall <[e-mail]>
 # Copyright 2006, 2007 Alexis S. L. Carvalho <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import base64
 import os
 import socket
 
 from .i18n import _
 from . import (
     encoding,
     error,
     httpconnection as httpconnectionmod,
     keepalive,
     pycompat,
     sslutil,
+    urllibcompat,
     util,
 )
 
 httplib = util.httplib
 stringio = util.stringio
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 class passwordmgr(object):
     def __init__(self, ui, passwddb):
         self.ui = ui
         self.passwddb = passwddb
 
     def add_password(self, realm, uri, user, passwd):
         return self.passwddb.add_password(realm, uri, user, passwd)
 
     def find_user_password(self, realm, authuri):
         authinfo = self.passwddb.find_user_password(realm, authuri)
         user, passwd = authinfo
         if user and passwd:
             self._writedebug(user, passwd)
             return (user, passwd)
 
         if not user or not passwd:
             res = httpconnectionmod.readauthforuri(self.ui, authuri, user)
             if res:
                 group, auth = res
                 user, passwd = auth.get('username'), auth.get('password')
                 self.ui.debug("using auth.%s.* for authentication\n" % group)
         if not user or not passwd:
             u = util.url(authuri)
             u.query = None
             if not self.ui.interactive():
                 raise error.Abort(_('http authorization required for %s') %
                                  util.hidepassword(str(u)))
 
             self.ui.write(_("http authorization required for %s\n") %
                           util.hidepassword(str(u)))
             self.ui.write(_("realm: %s\n") % realm)
             if user:
                 self.ui.write(_("user: %s\n") % user)
             else:
                 user = self.ui.prompt(_("user:"), default=None)
 
             if not passwd:
                 passwd = self.ui.getpass()
 
         self.passwddb.add_password(realm, authuri, user, passwd)
         self._writedebug(user, passwd)
         return (user, passwd)
 
     def _writedebug(self, user, passwd):
         msg = _('http auth: user %s, password %s\n')
         self.ui.debug(msg % (user, passwd and '*' * len(passwd) or 'not set'))
 
     def find_stored_password(self, authuri):
         return self.passwddb.find_user_password(None, authuri)
 
 class proxyhandler(urlreq.proxyhandler):
     def __init__(self, ui):
         proxyurl = (ui.config("http_proxy", "host") or
                         encoding.environ.get('http_proxy'))
         # XXX proxyauthinfo = None
 
         if proxyurl:
             # proxy can be proper url or host[:port]
             if not (proxyurl.startswith('http:') or
                     proxyurl.startswith('https:')):
                 proxyurl = 'http://' + proxyurl + '/'
             proxy = util.url(proxyurl)
             if not proxy.user:
                 proxy.user = ui.config("http_proxy", "user")
                 proxy.passwd = ui.config("http_proxy", "passwd")
 
             # see if we should use a proxy for this url
             no_list = ["localhost", "127.0.0.1"]
             no_list.extend([p.lower() for
                             p in ui.configlist("http_proxy", "no")])
             no_list.extend([p.strip().lower() for
                             p in encoding.environ.get("no_proxy", '').split(',')
                             if p.strip()])
             # "http_proxy.always" config is for running tests on localhost
             if ui.configbool("http_proxy", "always"):
                 self.no_list = []
             else:
                 self.no_list = no_list
 
             proxyurl = str(proxy)
             proxies = {'http': proxyurl, 'https': proxyurl}
             ui.debug('proxying through http://%s:%s\n' %
                       (proxy.host, proxy.port))
         else:
             proxies = {}
 
         urlreq.proxyhandler.__init__(self, proxies)
         self.ui = ui
 
     def proxy_open(self, req, proxy, type_):
-        host = req.get_host().split(':')[0]
+        host = urllibcompat.gethost(req).split(':')[0]
         for e in self.no_list:
             if host == e:
                 return None
             if e.startswith('*.') and host.endswith(e[2:]):
                 return None
             if e.startswith('.') and host.endswith(e[1:]):
                 return None
 
         return urlreq.proxyhandler.proxy_open(self, req, proxy, type_)
 
 def _gen_sendfile(orgsend):
     def _sendfile(self, data):
         # send a file
         if isinstance(data, httpconnectionmod.httpsendfile):
             # if auth required, some data sent twice, so rewind here
             data.seek(0)
             for chunk in util.filechunkiter(data):
                 orgsend(self, chunk)
         else:
             orgsend(self, data)
     return _sendfile
 
 has_https = util.safehasattr(urlreq, 'httpshandler')
 
 class httpconnection(keepalive.HTTPConnection):
     # must be able to send big bundle as stream.
     send = _gen_sendfile(keepalive.HTTPConnection.send)
 
     def getresponse(self):
         proxyres = getattr(self, 'proxyres', None)
         if proxyres:
             if proxyres.will_close:
                 self.close()
             self.proxyres = None
             return proxyres
         return keepalive.HTTPConnection.getresponse(self)
 
 # general transaction handler to support different ways to handle
 # HTTPS proxying before and after Python 2.6.3.
 def _generic_start_transaction(handler, h, req):
     tunnel_host = getattr(req, '_tunnel_host', None)
     if tunnel_host:
         if tunnel_host[:7] not in ['http://', 'https:/']:
             tunnel_host = 'https://' + tunnel_host
         new_tunnel = True
     else:
-        tunnel_host = req.get_selector()
+        tunnel_host = urllibcompat.getselector(req)
         new_tunnel = False
 
-    if new_tunnel or tunnel_host == req.get_full_url(): # has proxy
+    if new_tunnel or tunnel_host == urllibcompat.getfullurl(req): # has proxy
         u = util.url(tunnel_host)
         if new_tunnel or u.scheme == 'https': # only use CONNECT for HTTPS
             h.realhostport = ':'.join([u.host, (u.port or '443')])
             h.headers = req.headers.copy()
             h.headers.update(handler.parent.addheaders)
             return
 
     h.realhostport = None
     h.headers = None
 
 def _generic_proxytunnel(self):
     proxyheaders = dict(
             [(x, self.headers[x]) for x in self.headers
              if x.lower().startswith('proxy-')])
     self.send('CONNECT %s HTTP/1.0\r\n' % self.realhostport)
     for header in proxyheaders.iteritems():
         self.send('%s: %s\r\n' % header)
     self.send('\r\n')
 
     # majority of the following code is duplicated from
     # httplib.HTTPConnection as there are no adequate places to
     # override functions to provide the needed functionality
     res = self.response_class(self.sock,
                               strict=self.strict,
                               method=self._method)
 
     while True:
         version, status, reason = res._read_status()
         if status != httplib.CONTINUE:
             break
         # skip lines that are all whitespace
         list(iter(lambda: res.fp.readline().strip(), ''))
     res.status = status
     res.reason = reason.strip()
 
     if res.status == 200:
         # skip lines until we find a blank line
         list(iter(res.fp.readline, '\r\n'))
         return True
 
     if version == 'HTTP/1.0':
         res.version = 10
     elif version.startswith('HTTP/1.'):
         res.version = 11
     elif version == 'HTTP/0.9':
         res.version = 9
     else:
         raise httplib.UnknownProtocol(version)
 
     if res.version == 9:
         res.length = None
         res.chunked = 0
         res.will_close = 1
         res.msg = httplib.HTTPMessage(stringio())
         return False
 
     res.msg = httplib.HTTPMessage(res.fp)
     res.msg.fp = None
 
     # are we using the chunked-style of transfer encoding?
     trenc = res.msg.getheader('transfer-encoding')
     if trenc and trenc.lower() == "chunked":
         res.chunked = 1
         res.chunk_left = None
     else:
         res.chunked = 0
 
     # will the connection close at the end of the response?
     res.will_close = res._check_close()
 
     # do we have a Content-Length?
     # NOTE: RFC 2616, section 4.4, #3 says we ignore this if
     # transfer-encoding is "chunked"
     length = res.msg.getheader('content-length')
     if length and not res.chunked:
         try:
             res.length = int(length)
         except ValueError:
             res.length = None
         else:
             if res.length < 0:  # ignore nonsensical negative lengths
                 res.length = None
     else:
         res.length = None
 
     # does the body have a fixed length? (of zero)
     if (status == httplib.NO_CONTENT or status == httplib.NOT_MODIFIED or
         100 <= status < 200 or # 1xx codes
         res._method == 'HEAD'):
         res.length = 0
 
     # if the connection remains open, and we aren't using chunked, and
     # a content-length was not provided, then assume that the connection
     # WILL close.
     if (not res.will_close and
        not res.chunked and
        res.length is None):
         res.will_close = 1
 
     self.proxyres = res
 
     return False
 
 class httphandler(keepalive.HTTPHandler):
     def http_open(self, req):
         return self.do_open(httpconnection, req)
 
     def _start_transaction(self, h, req):
         _generic_start_transaction(self, h, req)
         return keepalive.HTTPHandler._start_transaction(self, h, req)
 
 if has_https:
     class httpsconnection(httplib.HTTPConnection):
         response_class = keepalive.HTTPResponse
         default_port = httplib.HTTPS_PORT
         # must be able to send big bundle as stream.
         send = _gen_sendfile(keepalive.safesend)
         getresponse = keepalive.wrapgetresponse(httplib.HTTPConnection)
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      *args, **kwargs):
             httplib.HTTPConnection.__init__(self, host, port, *args, **kwargs)
             self.key_file = key_file
             self.cert_file = cert_file
 
         def connect(self):
             self.sock = socket.create_connection((self.host, self.port))
 
             host = self.host
             if self.realhostport: # use CONNECT proxy
                 _generic_proxytunnel(self)
                 host = self.realhostport.rsplit(':', 1)[0]
             self.sock = sslutil.wrapsocket(
                 self.sock, self.key_file, self.cert_file, ui=self.ui,
                 serverhostname=host)
             sslutil.validatesocket(self.sock)
 
     class httpshandler(keepalive.KeepAliveHandler, urlreq.httpshandler):
         def __init__(self, ui):
             keepalive.KeepAliveHandler.__init__(self)
             urlreq.httpshandler.__init__(self)
             self.ui = ui
             self.pwmgr = passwordmgr(self.ui,
                                      self.ui.httppasswordmgrdb)
 
         def _start_transaction(self, h, req):
             _generic_start_transaction(self, h, req)
             return keepalive.KeepAliveHandler._start_transaction(self, h, req)
 
         def https_open(self, req):
-            # req.get_full_url() does not contain credentials and we may
-            # need them to match the certificates.
-            url = req.get_full_url()
+            # urllibcompat.getfullurl() does not contain credentials
+            # and we may need them to match the certificates.
+            url = urllibcompat.getfullurl(req)
             user, password = self.pwmgr.find_stored_password(url)
             res = httpconnectionmod.readauthforuri(self.ui, url, user)
             if res:
                 group, auth = res
                 self.auth = auth
                 self.ui.debug("using auth.%s.* for authentication\n" % group)
             else:
                 self.auth = None
             return self.do_open(self._makeconnection, req)
 
         def _makeconnection(self, host, port=None, *args, **kwargs):
             keyfile = None
             certfile = None
 
             if len(args) >= 1: # key_file
                 keyfile = args[0]
             if len(args) >= 2: # cert_file
                 certfile = args[1]
             args = args[2:]
 
             # if the user has specified different key/cert files in
             # hgrc, we prefer these
             if self.auth and 'key' in self.auth and 'cert' in self.auth:
                 keyfile = self.auth['key']
                 certfile = self.auth['cert']
 
             conn = httpsconnection(host, port, keyfile, certfile, *args,
                                    **kwargs)
             conn.ui = self.ui
             return conn
 
 class httpdigestauthhandler(urlreq.httpdigestauthhandler):
     def __init__(self, *args, **kwargs):
         urlreq.httpdigestauthhandler.__init__(self, *args, **kwargs)
         self.retried_req = None
 
     def reset_retry_count(self):
         # Python 2.6.5 will call this on 401 or 407 errors and thus loop
         # forever. We disable reset_retry_count completely and reset in
         # http_error_auth_reqed instead.
         pass
 
     def http_error_auth_reqed(self, auth_header, host, req, headers):
         # Reset the retry counter once for each request.
         if req is not self.retried_req:
             self.retried_req = req
             self.retried = 0
         return urlreq.httpdigestauthhandler.http_error_auth_reqed(
                     self, auth_header, host, req, headers)
 
 class httpbasicauthhandler(urlreq.httpbasicauthhandler):
     def __init__(self, *args, **kwargs):
         self.auth = None
         urlreq.httpbasicauthhandler.__init__(self, *args, **kwargs)
         self.retried_req = None
 
     def http_request(self, request):
         if self.auth:
             request.add_unredirected_header(self.auth_header, self.auth)
 
         return request
 
     def https_request(self, request):
         if self.auth:
             request.add_unredirected_header(self.auth_header, self.auth)
 
         return request
 
     def reset_retry_count(self):
         # Python 2.6.5 will call this on 401 or 407 errors and thus loop
         # forever. We disable reset_retry_count completely and reset in
         # http_error_auth_reqed instead.
         pass
 
     def http_error_auth_reqed(self, auth_header, host, req, headers):
         # Reset the retry counter once for each request.
         if req is not self.retried_req:
             self.retried_req = req
             self.retried = 0
         return urlreq.httpbasicauthhandler.http_error_auth_reqed(
                         self, auth_header, host, req, headers)
 
     def retry_http_basic_auth(self, host, req, realm):
-        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        user, pw = self.passwd.find_user_password(
+            realm, urllibcompat.getfullurl(req))
         if pw is not None:
             raw = "%s:%s" % (user, pw)
             auth = 'Basic %s' % base64.b64encode(raw).strip()
             if req.get_header(self.auth_header, None) == auth:
                 return None
             self.auth = auth
             req.add_unredirected_header(self.auth_header, auth)
             return self.parent.open(req)
         else:
             return None
 
 class cookiehandler(urlreq.basehandler):
     def __init__(self, ui):
         self.cookiejar = None
 
         cookiefile = ui.config('auth', 'cookiefile')
         if not cookiefile:
             return
 
         cookiefile = util.expandpath(cookiefile)
         try:
             cookiejar = util.cookielib.MozillaCookieJar(cookiefile)
             cookiejar.load()
             self.cookiejar = cookiejar
         except util.cookielib.LoadError as e:
             ui.warn(_('(error loading cookie file %s: %s; continuing without '
                       'cookies)\n') % (cookiefile, str(e)))
 
     def http_request(self, request):
         if self.cookiejar:
             self.cookiejar.add_cookie_header(request)
 
         return request
 
     def https_request(self, request):
         if self.cookiejar:
             self.cookiejar.add_cookie_header(request)
 
         return request
 
 handlerfuncs = []
 
 def opener(ui, authinfo=None):
     '''
     construct an opener suitable for urllib2
     authinfo will be added to the password manager
     '''
     # experimental config: ui.usehttp2
     if ui.configbool('ui', 'usehttp2'):
         handlers = [
             httpconnectionmod.http2handler(
                 ui,
                 passwordmgr(ui, ui.httppasswordmgrdb))
         ]
     else:
         handlers = [httphandler()]
         if has_https:
             handlers.append(httpshandler(ui))
 
     handlers.append(proxyhandler(ui))
 
     passmgr = passwordmgr(ui, ui.httppasswordmgrdb)
     if authinfo is not None:
         realm, uris, user, passwd = authinfo
         saveduser, savedpass = passmgr.find_stored_password(uris[0])
         if user != saveduser or passwd:
             passmgr.add_password(realm, uris, user, passwd)
         ui.debug('http auth: user %s, password %s\n' %
                  (user, passwd and '*' * len(passwd) or 'not set'))
 
     handlers.extend((httpbasicauthhandler(passmgr),
                      httpdigestauthhandler(passmgr)))
     handlers.extend([h(ui, passmgr) for h in handlerfuncs])
     handlers.append(cookiehandler(ui))
     opener = urlreq.buildopener(*handlers)
 
     # The user agent should should *NOT* be used by servers for e.g.
     # protocol detection or feature negotiation: there are other
     # facilities for that.
     #
     # "mercurial/proto-1.0" was the original user agent string and
     # exists for backwards compatibility reasons.
     #
     # The "(Mercurial %s)" string contains the distribution
     # name and version. Other client implementations should choose their
     # own distribution name. Since servers should not be using the user
     # agent string for anything, clients should be able to define whatever
     # user agent they deem appropriate.
     agent = 'mercurial/proto-1.0 (Mercurial %s)' % util.version()
     opener.addheaders = [(r'User-agent', pycompat.sysstr(agent))]
 
     # This header should only be needed by wire protocol requests. But it has
     # been sent on all requests since forever. We keep sending it for backwards
     # compatibility reasons. Modern versions of the wire protocol use
     # X-HgProto-<N> for advertising client support.
     opener.addheaders.append((r'Accept', r'application/mercurial-0.1'))
     return opener
 
 def open(ui, url_, data=None):
     u = util.url(url_)
     if u.scheme:
         u.scheme = u.scheme.lower()
         url_, authinfo = u.authinfo()
     else:
         path = util.normpath(os.path.abspath(url_))
         url_ = 'file://' + urlreq.pathname2url(path)
         authinfo = None
     return opener(ui, authinfo).open(url_, data)