largefiles: send statlfile remote calls only for nonexisting locally files

Files that are already in local store should be checked locally. The problem
with this implementation is how difference in messages between local and remote
checks should look like. For now local errors for file missing and content
corrupted looks like this:

'changeset cset: filename references missing storepath\n'
'changeset cset: filename references corrupted storepath\n'

for remote it looks like:

'changeset cset: filename missing\n'
'changeset cset: filename: contents differ\n'

Contents differ error for remote calls is never raised currently - for now
statlfile implementation lacks checking file content.

hgext/largefiles/remotestore.py

@@ -1,108 +1,125 b''
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''remote largefile store; the base class for wirestore'''
 
 from mercurial import util, wireproto, error
 from mercurial.i18n import _
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 import lfutil
 import basestore
+import localstore
 
 class remotestore(basestore.basestore):
     '''a largefile store accessed over a network'''
     def __init__(self, ui, repo, url):
         super(remotestore, self).__init__(ui, repo, url)
+        self._lstore = localstore.localstore(self.ui, self.repo, self.repo)
 
     def put(self, source, hash):
         if self.sendfile(source, hash):
             raise error.Abort(
                 _('remotestore: could not put %s to remote store %s')
                 % (source, util.hidepassword(self.url)))
         self.ui.debug(
             _('remotestore: put %s to remote store %s\n')
             % (source, util.hidepassword(self.url)))
 
     def exists(self, hashes):
         return dict((h, s == 0) for (h, s) in # dict-from-generator
                     self._stat(hashes).iteritems())
 
     def sendfile(self, filename, hash):
         self.ui.debug('remotestore: sendfile(%s, %s)\n' % (filename, hash))
         fd = None
         try:
             fd = lfutil.httpsendfile(self.ui, filename)
             return self._put(hash, fd)
         except IOError as e:
             raise error.Abort(
                 _('remotestore: could not open file %s: %s')
                 % (filename, str(e)))
         finally:
             if fd:
                 fd.close()
 
     def _getfile(self, tmpfile, filename, hash):
         try:
             chunks = self._get(hash)
         except urlerr.httperror as e:
             # 401s get converted to error.Aborts; everything else is fine being
             # turned into a StoreError
             raise basestore.StoreError(filename, hash, self.url, str(e))
         except urlerr.urlerror as e:
             # This usually indicates a connection problem, so don't
             # keep trying with the other files... they will probably
             # all fail too.
             raise error.Abort('%s: %s' %
                              (util.hidepassword(self.url), e.reason))
         except IOError as e:
             raise basestore.StoreError(filename, hash, self.url, str(e))
 
         return lfutil.copyandhash(chunks, tmpfile)
 
+    def _hashesavailablelocally(self, hashes):
+        existslocallymap = self._lstore.exists(hashes)
+        localhashes = [hash for hash in hashes if existslocallymap[hash]]
+        return localhashes
+
     def _verifyfiles(self, contents, filestocheck):
         failed = False
         expectedhashes = [expectedhash
                           for cset, filename, expectedhash in filestocheck]
-        stats = self._stat(expectedhashes)
+        localhashes = self._hashesavailablelocally(expectedhashes)
+        stats = self._stat([expectedhash for expectedhash in expectedhashes
+                            if expectedhash not in localhashes])
+
         for cset, filename, expectedhash in filestocheck:
+            if expectedhash in localhashes:
+                filetocheck = (cset, filename, expectedhash)
+                verifyresult = self._lstore._verifyfiles(contents,
+                                                         [filetocheck])
+                if verifyresult:
+                    failed = True
+            else:
             stat = stats[expectedhash]
             if stat:
                 if stat == 1:
                     self.ui.warn(
                         _('changeset %s: %s: contents differ\n')
                         % (cset, filename))
                     failed = True
                 elif stat == 2:
                     self.ui.warn(
                         _('changeset %s: %s missing\n')
                         % (cset, filename))
                     failed = True
                 else:
                     raise RuntimeError('verify failed: unexpected response '
                                        'from statlfile (%r)' % stat)
         return failed
 
     def batch(self):
         '''Support for remote batching.'''
         return wireproto.remotebatch(self)
 
     def _put(self, hash, fd):
         '''Put file with the given hash in the remote store.'''
         raise NotImplementedError('abstract method')
 
     def _get(self, hash):
         '''Get file with the given hash from the remote store.'''
         raise NotImplementedError('abstract method')
 
     def _stat(self, hashes):
         '''Get information about availability of files specified by
         hashes in the remote store. Return dictionary mapping hashes
         to return code where 0 means that file is available, other
         values if not.'''
         raise NotImplementedError('abstract method')

tests/test-largefiles-wireproto.t

@@ -1,351 +1,383 b''
 This file contains testcases that tend to be related to the wire protocol part
 of largefiles.
 
   $ USERCACHE="$TESTTMP/cache"; export USERCACHE
   $ mkdir "${USERCACHE}"
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > largefiles=
   > purge=
   > rebase=
   > transplant=
   > [phases]
   > publish=False
   > [largefiles]
   > minsize=2
   > patterns=glob:**.dat
   > usercache=${USERCACHE}
   > [web]
   > allow_archive = zip
   > [hooks]
   > precommit=sh -c "echo \\"Invoking status precommit hook\\"; hg status"
   > EOF
 
 
 #if serve
 vanilla clients not locked out from largefiles servers on vanilla repos
   $ mkdir r1
   $ cd r1
   $ hg init
   $ echo c1 > f1
   $ hg add f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cd ..
   $ hg serve -R r1 -d -p $HGPORT --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg --config extensions.largefiles=! clone http://localhost:$HGPORT r2
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 largefiles clients still work with vanilla servers
   $ hg serve --config extensions.largefiles=! -R r1 -d -p $HGPORT1 --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg clone http://localhost:$HGPORT1 r3
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 #endif
 
 vanilla clients locked out from largefiles http repos
   $ mkdir r4
   $ cd r4
   $ hg init
   $ echo c1 > f1
   $ hg add --large f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cd ..
 
 largefiles can be pushed locally (issue3583)
   $ hg init dest
   $ cd r4
   $ hg outgoing ../dest
   comparing with ../dest
   searching for changes
   changeset:   0:639881c12b4c
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     m1
   
   $ hg push ../dest
   pushing to ../dest
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
 
 exit code with nothing outgoing (issue3611)
   $ hg outgoing ../dest
   comparing with ../dest
   searching for changes
   no changes found
   [1]
   $ cd ..
 
 #if serve
   $ hg serve -R r4 -d -p $HGPORT2 --pid-file hg.pid
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg --config extensions.largefiles=! clone http://localhost:$HGPORT2 r5
   abort: remote error:
   
   This repository uses the largefiles extension.
   
   Please enable it in your Mercurial config file.
   [255]
 
 used all HGPORTs, kill all daemons
   $ killdaemons.py
 #endif
 
 vanilla clients locked out from largefiles ssh repos
   $ hg --config extensions.largefiles=! clone -e "python \"$TESTDIR/dummyssh\"" ssh://user@dummy/r4 r5
   remote: 
   remote: This repository uses the largefiles extension.
   remote: 
   remote: Please enable it in your Mercurial config file.
   remote: 
   remote: -
   abort: remote error
   (check previous remote output)
   [255]
 
 #if serve
 
 largefiles clients refuse to push largefiles repos to vanilla servers
   $ mkdir r6
   $ cd r6
   $ hg init
   $ echo c1 > f1
   $ hg add f1
   $ hg commit -m "m1"
   Invoking status precommit hook
   A f1
   $ cat >> .hg/hgrc <<!
   > [web]
   > push_ssl = false
   > allow_push = *
   > !
   $ cd ..
   $ hg clone r6 r7
   updating to branch default
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ cd r7
   $ echo c2 > f2
   $ hg add --large f2
   $ hg commit -m "m2"
   Invoking status precommit hook
   A f2
   $ hg serve --config extensions.largefiles=! -R ../r6 -d -p $HGPORT --pid-file ../hg.pid
   $ cat ../hg.pid >> $DAEMON_PIDS
   $ hg push http://localhost:$HGPORT
   pushing to http://localhost:$HGPORT/
   searching for changes
   abort: http://localhost:$HGPORT/ does not appear to be a largefile store
   [255]
   $ cd ..
 
 putlfile errors are shown (issue3123)
 Corrupt the cached largefile in r7 and move it out of the servers usercache
   $ mv r7/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8 .
   $ echo 'client side corruption' > r7/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8
   $ rm "$USERCACHE/4cdac4d8b084d0b599525cf732437fb337d422a8"
   $ hg init empty
   $ hg serve -R empty -d -p $HGPORT1 --pid-file hg.pid \
   >   --config 'web.allow_push=*' --config web.push_ssl=False
   $ cat hg.pid >> $DAEMON_PIDS
   $ hg push -R r7 http://localhost:$HGPORT1
   pushing to http://localhost:$HGPORT1/
   searching for changes
   remote: largefiles: failed to put 4cdac4d8b084d0b599525cf732437fb337d422a8 into store: largefile contents do not match hash
   abort: remotestore: could not put $TESTTMP/r7/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8 to remote store http://localhost:$HGPORT1/ (glob)
   [255]
   $ mv 4cdac4d8b084d0b599525cf732437fb337d422a8 r7/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8
 Push of file that exists on server but is corrupted - magic healing would be nice ... but too magic
   $ echo "server side corruption" > empty/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8
   $ hg push -R r7 http://localhost:$HGPORT1
   pushing to http://localhost:$HGPORT1/
   searching for changes
   remote: adding changesets
   remote: adding manifests
   remote: adding file changes
   remote: added 2 changesets with 2 changes to 2 files
   $ cat empty/.hg/largefiles/4cdac4d8b084d0b599525cf732437fb337d422a8
   server side corruption
   $ rm -rf empty
 
 Push a largefiles repository to a served empty repository
   $ hg init r8
   $ echo c3 > r8/f1
   $ hg add --large r8/f1 -R r8
   $ hg commit -m "m1" -R r8
   Invoking status precommit hook
   A f1
   $ hg init empty
   $ hg serve -R empty -d -p $HGPORT2 --pid-file hg.pid \
   >   --config 'web.allow_push=*' --config web.push_ssl=False
   $ cat hg.pid >> $DAEMON_PIDS
   $ rm "${USERCACHE}"/*
   $ hg push -R r8 http://localhost:$HGPORT2/#default
   pushing to http://localhost:$HGPORT2/
   searching for changes
   remote: adding changesets
   remote: adding manifests
   remote: adding file changes
   remote: added 1 changesets with 1 changes to 1 files
   $ [ -f "${USERCACHE}"/02a439e5c31c526465ab1a0ca1f431f76b827b90 ]
   $ [ -f empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 ]
 
 Clone over http, no largefiles pulled on clone.
 
   $ hg clone http://localhost:$HGPORT2/#default http-clone -U
   adding changesets
   adding manifests
   adding file changes
   added 1 changesets with 1 changes to 1 files
 
 Archive contains largefiles
   >>> import urllib2, os
   >>> u = 'http://localhost:%s/archive/default.zip' % os.environ['HGPORT2']
   >>> with open('archive.zip', 'w') as f:
   ...     f.write(urllib2.urlopen(u).read())
   $ unzip -t archive.zip
   Archive:  archive.zip
       testing: empty-default/.hg_archival.txt   OK
       testing: empty-default/f1         OK
   No errors detected in compressed data of archive.zip.
 
 test 'verify' with remotestore:
 
   $ rm "${USERCACHE}"/02a439e5c31c526465ab1a0ca1f431f76b827b90
   $ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 .
   $ hg -R http-clone verify --large --lfa
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   1 files, 1 changesets, 1 total revisions
   searching 1 changesets for largefiles
   changeset 0:cf03e5bb9936: f1 missing
   verified existence of 1 revisions of 1 largefiles
   [1]
   $ mv 02a439e5c31c526465ab1a0ca1f431f76b827b90 empty/.hg/largefiles/
   $ hg -R http-clone -q verify --large --lfa
 
 largefiles pulled on update - a largefile missing on the server:
   $ mv empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 .
   $ hg -R http-clone up --config largefiles.usercache=http-clone-usercache
   getting changed largefiles
   f1: largefile 02a439e5c31c526465ab1a0ca1f431f76b827b90 not available from http://localhost:$HGPORT2/
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg -R http-clone st
   ! f1
   $ hg -R http-clone up -Cqr null
 
 largefiles pulled on update - a largefile corrupted on the server:
   $ echo corruption > empty/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90
   $ hg -R http-clone up --config largefiles.usercache=http-clone-usercache
   getting changed largefiles
   f1: data corruption (expected 02a439e5c31c526465ab1a0ca1f431f76b827b90, got 6a7bb2556144babe3899b25e5428123735bb1e27)
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg -R http-clone st
   ! f1
   $ [ ! -f http-clone/.hg/largefiles/02a439e5c31c526465ab1a0ca1f431f76b827b90 ]
   $ [ ! -f http-clone/f1 ]
   $ [ ! -f http-clone-usercache ]
   $ hg -R http-clone verify --large --lfc
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   1 files, 1 changesets, 1 total revisions
   searching 1 changesets for largefiles
   verified contents of 1 revisions of 1 largefiles
   $ hg -R http-clone up -Cqr null
 
 largefiles pulled on update - no server side problems:
   $ mv 02a439e5c31c526465ab1a0ca1f431f76b827b90 empty/.hg/largefiles/
   $ hg -R http-clone --debug up --config largefiles.usercache=http-clone-usercache --config progress.debug=true
   resolving manifests
    branchmerge: False, force: False, partial: False
    ancestor: 000000000000, local: 000000000000+, remote: cf03e5bb9936
    .hglf/f1: remote created -> g
   getting .hglf/f1
   updating: .hglf/f1 1/1 files (100.00%)
   getting changed largefiles
   using http://localhost:$HGPORT2/
   sending capabilities command
   sending batch command
   getting largefiles: 0/1 files (0.00%)
   getting f1:02a439e5c31c526465ab1a0ca1f431f76b827b90
   sending getlfile command
   found 02a439e5c31c526465ab1a0ca1f431f76b827b90 in store
   1 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
   $ ls http-clone-usercache/*
   http-clone-usercache/02a439e5c31c526465ab1a0ca1f431f76b827b90
 
   $ rm -rf empty http-clone*
 
 used all HGPORTs, kill all daemons
   $ killdaemons.py
 
 largefiles should batch verify remote calls
 
   $ hg init batchverifymain
   $ cd batchverifymain
   $ echo "aaa" >> a
   $ hg add --large a
   $ hg commit -m "a"
   Invoking status precommit hook
   A a
   $ echo "bbb" >> b
   $ hg add --large b
   $ hg commit -m "b"
   Invoking status precommit hook
   A b
   $ cd ..
   $ hg serve -R batchverifymain -d -p $HGPORT --pid-file hg.pid \
   > -A access.log
   $ cat hg.pid >> $DAEMON_PIDS
-  $ hg clone http://localhost:$HGPORT batchverifyclone
+  $ hg clone --noupdate http://localhost:$HGPORT batchverifyclone
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 2 changes to 2 files
-  updating to branch default
-  getting changed largefiles
-  2 largefiles updated, 0 removed
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  $ hg -R batchverifyclone verify --large
+  $ hg -R batchverifyclone verify --large --lfa
   checking changesets
   checking manifests
   crosschecking files in changesets and manifests
   checking files
   2 files, 2 changesets, 2 total revisions
-  searching 1 changesets for largefiles
+  searching 2 changesets for largefiles
   verified existence of 2 revisions of 2 largefiles
   $ tail -1 access.log
   127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a (glob)
-  $ rm access.log
+  $ hg -R batchverifyclone update
+  getting changed largefiles
+  2 largefiles updated, 0 removed
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+
+Clear log file before next test
+
+  $ printf "" > access.log
+
+Verify should check file on remote server only when file is not
+available locally.
+
+  $ echo "ccc" >> batchverifymain/c
+  $ hg -R batchverifymain status
+  ? c
+  $ hg -R batchverifymain add --large batchverifymain/c
+  $ hg -R batchverifymain commit -m "c"
+  Invoking status precommit hook
+  A c
+  $ hg -R batchverifyclone pull
+  pulling from http://localhost:$HGPORT/
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  (run 'hg update' to get a working copy)
+  $ hg -R batchverifyclone verify --lfa
+  checking changesets
+  checking manifests
+  crosschecking files in changesets and manifests
+  checking files
+  3 files, 3 changesets, 3 total revisions
+  searching 3 changesets for largefiles
+  verified existence of 3 revisions of 3 largefiles
+  $ tail -1 access.log
+  127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c (glob)
 
   $ killdaemons.py
 
 #endif