largefiles: stat all largefiles in one batch before downloading

This avoids a lot of expensive roundtrips to remote repositories ... but might
be slightly slower for local operations.

This will also change some aborts on missing files to warnings. That will in
some situations make it possible to continue working on a repository with
missing largefiles.

hgext/largefiles/basestore.py

@@ -62,12 +62,19 b' class basestore(object):'
         util.makedirs(lfutil.storepath(self.repo, ''))
 
         at = 0
+        available = self.exists(set(hash for (_filename, hash) in files))
         for filename, hash in files:
             ui.progress(_('getting largefiles'), at, unit='lfile',
                 total=len(files))
             at += 1
             ui.note(_('getting %s:%s\n') % (filename, hash))
 
+            if not available.get(hash):
+                ui.warn(_('%s: largefile %s not available from %s\n')
+                        % (filename, hash, self.url))
+                missing.append(filename)
+                continue
+
             storefilename = lfutil.storepath(self.repo, hash)
             tmpfile = util.atomictempfile(storefilename + '.tmp',
                                           createmode=self.repo.store.createmode)

hgext/largefiles/remotestore.py

@@ -47,16 +47,6 b' class remotestore(basestore.basestore):'
                 fd.close()
 
     def _getfile(self, tmpfile, filename, hash):
-        # quit if the largefile isn't there
-        stat = self._stat([hash])[hash]
-        if stat == 1:
-            raise util.Abort(_('remotestore: largefile %s is invalid') % hash)
-        elif stat == 2:
-            raise util.Abort(_('remotestore: largefile %s is missing') % hash)
-        elif stat != 0:
-            raise RuntimeError('error getting file: unexpected response from '
-                               'statlfile (%r)' % stat)
-
         try:
             chunks = self._get(hash)
         except urllib2.HTTPError, e:

hgext/largefiles/wirestore.py

@@ -26,8 +26,9 b' class wirestore(remotestore.remotestore)'
         return self.remote.getlfile(hash)
 
     def _stat(self, hashes):
-        '''For each hash, return 2 if the largefile is missing, 1 if it has a
-        mismatched checksum, or 0 if it is in good condition'''
+        '''For each hash, return 0 if it is available, other values if not.
+        It is usually 2 if the largefile is missing, but might be 1 the server
+        has a corrupted copy.'''
         batch = self.remote.batch()
         futures = {}
         for hash in hashes:

tests/test-largefiles-cache.t

@@ -47,7 +47,7 b' but there is no cache file for it. So, '
 
   $ hg update -r0
   getting changed largefiles
-  error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+  large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status
@@ -64,7 +64,7 b' Update working directory to tip, again.'
 
   $ hg update -r0
   getting changed largefiles
-  error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
+  large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status

tests/test-largefiles.t

@@ -1306,7 +1306,7 b' Update to revision with missing largefil'
   $ rm ${USERCACHE}/7838695e10da2bb75ac1156565f40a2595fa2fa0
   $ hg up -r 6
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   1 largefiles updated, 2 removed
   4 files updated, 0 files merged, 2 files removed, 0 files unresolved
   $ rm normal3
@@ -1327,7 +1327,7 b' Update to revision with missing largefil'
   ! normal3
   $ hg up -Cr.
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg st
@@ -1349,7 +1349,7 b' Merge with revision with missing largefi'
   4 files updated, 0 files merged, 0 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
-  error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob)
+  large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d
   1 largefiles updated, 0 removed
 
   $ hg rollback -q
@@ -1773,8 +1773,11 b' largefiles pulled on update - a largefil'
   $ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 .
   $ hg -R http-clone up --config largefiles.usercache=http-clone-usercache
   getting changed largefiles
-  abort: remotestore: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 is missing
-  [255]
+  f1: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 not available from http://localhost:$HGPORT2/
+  0 largefiles updated, 0 removed
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ hg -R http-clone st
+  ! f1
   $ hg -R http-clone up -Cqr null
 
 largefiles pulled on update - a largefile corrupted on the server:
@@ -1811,9 +1814,9 b' largefiles pulled on update - no server '
   getting changed largefiles
   using http://localhost:$HGPORT2/
   sending capabilities command
+  sending batch command
   getting largefiles: 0/1 lfile (0.00%)
   getting f1:02a439e5c31c526465ab1a0ca1f431f76b827b90
-  sending batch command
   sending getlfile command
   found 02a439e5c31c526465ab1a0ca1f431f76b827b90 in store
   1 largefiles updated, 0 removed

tests/test-lfconvert.t

@@ -343,7 +343,7 b' Ensure the abort message is useful if a '
   $ rm largefiles-repo/.hg/largefiles/*
   $ hg lfconvert --to-normal issue3519 normalized3519
   initializing destination normalized3519
-  error getting id 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 from url file:$TESTTMP/largefiles-repo for file large: can't get file locally (glob)
+  large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:$TESTTMP/largefiles-repo
   abort: missing largefile 'large' from revision d4892ec57ce212905215fad1d9018f56b99202ad
   [255]