http: handle push of bundles > 2 GB again (issue3017)

It was very elegant that httpsendfile implemented __len__ like a string. It was
however also dangerous because that protocol can't handle sizes bigger than 2 GB.
Mercurial tried to work around that, but it turned out to be too easy to
introduce new errors in this area.

With this change __len__ is no longer implemented at all and the code will work
the same way for short and long posts.

mercurial/httpconnection.py

@@ -22,8 +22,9 b' from mercurial.i18n import _'
 class httpsendfile(object):
     """This is a wrapper around the objects returned by python's "open".
 
-    Its purpose is to send file-like objects via HTTP and, to do so, it
-    defines a __len__ attribute to feed the Content-Length header.
+    Its purpose is to send file-like objects via HTTP.
+    It do however not define a __len__ attribute because the length
+    might be more than Py_ssize_t can handle.
     """
 
     def __init__(self, ui, *args, **kwargs):
@@ -35,9 +36,9 b' class httpsendfile(object):'
         self.seek = self._data.seek
         self.close = self._data.close
         self.write = self._data.write
-        self._len = os.fstat(self._data.fileno()).st_size
+        self.length = os.fstat(self._data.fileno()).st_size
         self._pos = 0
-        self._total = self._len / 1024 * 2
+        self._total = self.length / 1024 * 2
 
     def read(self, *args, **kwargs):
         try:
@@ -54,9 +55,6 b' class httpsendfile(object):'
                          unit=_('kb'), total=self._total)
         return ret
 
-    def __len__(self):
-        return self._len
-
 # moved here from url.py to avoid a cycle
 def readauthforuri(ui, uri, user):
     # Read configuration

mercurial/httprepo.py

@@ -74,9 +74,14 b' class httprepository(wireproto.wirerepos'
         if cmd == 'pushkey':
             args['data'] = ''
         data = args.pop('data', None)
+        size = 0
+        if util.safehasattr(data, 'length'):
+            size = data.length
+        elif data is not None:
+            size = len(data)
         headers = args.pop('headers', {})
 
-        if data and self.ui.configbool('ui', 'usehttp2', False):
+        if size and self.ui.configbool('ui', 'usehttp2', False):
             headers['Expect'] = '100-Continue'
             headers['X-HgHttp2'] = '1'
 
@@ -105,9 +110,6 b' class httprepository(wireproto.wirerepos'
         cu = "%s%s" % (self._url, qs)
         req = urllib2.Request(cu, data, headers)
         if data is not None:
-            # len(data) is broken if data doesn't fit into Py_ssize_t
-            # add the header ourself to avoid OverflowError
-            size = data.__len__()
             self.ui.debug("sending %s bytes\n" % size)
             req.add_unredirected_header('Content-Length', '%d' % size)
         try:

mercurial/util.py

@@ -24,6 +24,10 b' import imp, socket, urllib'
 def sha1(s):
     return _fastsha1(s)
 
+_notset = object()
+def safehasattr(thing, attr):
+    return getattr(thing, attr, _notset) is not _notset
+
 def _fastsha1(s):
     # This function will import sha1 from hashlib or sha (whichever is
     # available) and overwrite itself with it on the first call.