# HG changeset patch # User Mads Kiilerich # Date 2013-04-15 21:37:43 # Node ID 9d33d6e0d442df89d514859306dda3a1a0bd2749 # Parent 266b5fb72f2655c17dd2f5074719df2b33026120 largefiles: stat all largefiles in one batch before downloading This avoids a lot of expensive roundtrips to remote repositories ... but might be slightly slower for local operations. This will also change some aborts on missing files to warnings. That will in some situations make it possible to continue working on a repository with missing largefiles. diff --git a/hgext/largefiles/basestore.py b/hgext/largefiles/basestore.py --- a/hgext/largefiles/basestore.py +++ b/hgext/largefiles/basestore.py @@ -62,12 +62,19 @@ class basestore(object): util.makedirs(lfutil.storepath(self.repo, '')) at = 0 + available = self.exists(set(hash for (_filename, hash) in files)) for filename, hash in files: ui.progress(_('getting largefiles'), at, unit='lfile', total=len(files)) at += 1 ui.note(_('getting %s:%s\n') % (filename, hash)) + if not available.get(hash): + ui.warn(_('%s: largefile %s not available from %s\n') + % (filename, hash, self.url)) + missing.append(filename) + continue + storefilename = lfutil.storepath(self.repo, hash) tmpfile = util.atomictempfile(storefilename + '.tmp', createmode=self.repo.store.createmode) diff --git a/hgext/largefiles/remotestore.py b/hgext/largefiles/remotestore.py --- a/hgext/largefiles/remotestore.py +++ b/hgext/largefiles/remotestore.py @@ -47,16 +47,6 @@ class remotestore(basestore.basestore): fd.close() def _getfile(self, tmpfile, filename, hash): - # quit if the largefile isn't there - stat = self._stat([hash])[hash] - if stat == 1: - raise util.Abort(_('remotestore: largefile %s is invalid') % hash) - elif stat == 2: - raise util.Abort(_('remotestore: largefile %s is missing') % hash) - elif stat != 0: - raise RuntimeError('error getting file: unexpected response from ' - 'statlfile (%r)' % stat) - try: chunks = self._get(hash) except urllib2.HTTPError, e: diff --git a/hgext/largefiles/wirestore.py b/hgext/largefiles/wirestore.py --- a/hgext/largefiles/wirestore.py +++ b/hgext/largefiles/wirestore.py @@ -26,8 +26,9 @@ class wirestore(remotestore.remotestore) return self.remote.getlfile(hash) def _stat(self, hashes): - '''For each hash, return 2 if the largefile is missing, 1 if it has a - mismatched checksum, or 0 if it is in good condition''' + '''For each hash, return 0 if it is available, other values if not. + It is usually 2 if the largefile is missing, but might be 1 the server + has a corrupted copy.''' batch = self.remote.batch() futures = {} for hash in hashes: diff --git a/tests/test-largefiles-cache.t b/tests/test-largefiles-cache.t --- a/tests/test-largefiles-cache.t +++ b/tests/test-largefiles-cache.t @@ -47,7 +47,7 @@ but there is no cache file for it. So, $ hg update -r0 getting changed largefiles - error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob) + large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror 0 largefiles updated, 0 removed 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg status @@ -64,7 +64,7 @@ Update working directory to tip, again. $ hg update -r0 getting changed largefiles - error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob) + large: largefile 7f7097b041ccf68cc5561e9600da4655d21c6d18 not available from file:$TESTTMP/mirror 0 largefiles updated, 0 removed 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg status diff --git a/tests/test-largefiles.t b/tests/test-largefiles.t --- a/tests/test-largefiles.t +++ b/tests/test-largefiles.t @@ -1306,7 +1306,7 @@ Update to revision with missing largefil $ rm ${USERCACHE}/7838695e10da2bb75ac1156565f40a2595fa2fa0 $ hg up -r 6 getting changed largefiles - error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob) + large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d 1 largefiles updated, 2 removed 4 files updated, 0 files merged, 2 files removed, 0 files unresolved $ rm normal3 @@ -1327,7 +1327,7 @@ Update to revision with missing largefil ! normal3 $ hg up -Cr. getting changed largefiles - error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob) + large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d 0 largefiles updated, 0 removed 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg st @@ -1349,7 +1349,7 @@ Merge with revision with missing largefi 4 files updated, 0 files merged, 0 files removed, 0 files unresolved (branch merge, don't forget to commit) getting changed largefiles - error getting id 7838695e10da2bb75ac1156565f40a2595fa2fa0 from url file:$TESTTMP/d for file large3: can't get file locally (glob) + large3: largefile 7838695e10da2bb75ac1156565f40a2595fa2fa0 not available from file:$TESTTMP/d 1 largefiles updated, 0 removed $ hg rollback -q @@ -1773,8 +1773,11 @@ largefiles pulled on update - a largefil $ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 . $ hg -R http-clone up --config largefiles.usercache=http-clone-usercache getting changed largefiles - abort: remotestore: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 is missing - [255] + f1: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 not available from http://localhost:$HGPORT2/ + 0 largefiles updated, 0 removed + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ hg -R http-clone st + ! f1 $ hg -R http-clone up -Cqr null largefiles pulled on update - a largefile corrupted on the server: @@ -1811,9 +1814,9 @@ largefiles pulled on update - no server getting changed largefiles using http://localhost:$HGPORT2/ sending capabilities command + sending batch command getting largefiles: 0/1 lfile (0.00%) getting f1:02a439e5c31c526465ab1a0ca1f431f76b827b90 - sending batch command sending getlfile command found 02a439e5c31c526465ab1a0ca1f431f76b827b90 in store 1 largefiles updated, 0 removed diff --git a/tests/test-lfconvert.t b/tests/test-lfconvert.t --- a/tests/test-lfconvert.t +++ b/tests/test-lfconvert.t @@ -343,7 +343,7 @@ Ensure the abort message is useful if a $ rm largefiles-repo/.hg/largefiles/* $ hg lfconvert --to-normal issue3519 normalized3519 initializing destination normalized3519 - error getting id 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 from url file:$TESTTMP/largefiles-repo for file large: can't get file locally (glob) + large: largefile 2e000fa7e85759c7f4c254d4d9c33ef481e459a7 not available from file:$TESTTMP/largefiles-repo abort: missing largefile 'large' from revision d4892ec57ce212905215fad1d9018f56b99202ad [255]