largefiles: getstandinmatcher should not depend on existence of directories

Looking for a (potentially empty) directory was not reliable - both because it
is a reasonable assumption that empty directories can be removed and because it
wasn't created in all cases ... such as when pulling to an existing repository.

hgext/largefiles/lfutil.py

@@ -1,431 +1,427 b''
 # Copyright 2009-2010 Gregory P. Ward
 # Copyright 2009-2010 Intelerad Medical Systems Incorporated
 # Copyright 2010-2011 Fog Creek Software
 # Copyright 2010-2011 Unity Technologies
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''largefiles utility code: must not import other modules in this package.'''
 
 import os
 import errno
 import platform
 import shutil
 import stat
 
 from mercurial import dirstate, httpconnection, match as match_, util, scmutil
 from mercurial.i18n import _
 
 shortname = '.hglf'
 shortnameslash = shortname + '/'
 longname = 'largefiles'
 
 
 # -- Private worker functions ------------------------------------------
 
 def getminsize(ui, assumelfiles, opt, default=10):
     lfsize = opt
     if not lfsize and assumelfiles:
         lfsize = ui.config(longname, 'minsize', default=default)
     if lfsize:
         try:
             lfsize = float(lfsize)
         except ValueError:
             raise util.Abort(_('largefiles: size must be number (not %s)\n')
                              % lfsize)
     if lfsize is None:
         raise util.Abort(_('minimum size for largefiles must be specified'))
     return lfsize
 
 def link(src, dest):
     try:
         util.oslink(src, dest)
     except OSError:
         # if hardlinks fail, fallback on atomic copy
         dst = util.atomictempfile(dest)
         for chunk in util.filechunkiter(open(src, 'rb')):
             dst.write(chunk)
         dst.close()
         os.chmod(dest, os.stat(src).st_mode)
 
 def usercachepath(ui, hash):
     path = ui.configpath(longname, 'usercache', None)
     if path:
         path = os.path.join(path, hash)
     else:
         if os.name == 'nt':
             appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA'))
             if appdata:
                 path = os.path.join(appdata, longname, hash)
         elif platform.system() == 'Darwin':
             home = os.getenv('HOME')
             if home:
                 path = os.path.join(home, 'Library', 'Caches',
                                     longname, hash)
         elif os.name == 'posix':
             path = os.getenv('XDG_CACHE_HOME')
             if path:
                 path = os.path.join(path, longname, hash)
             else:
                 home = os.getenv('HOME')
                 if home:
                     path = os.path.join(home, '.cache', longname, hash)
         else:
             raise util.Abort(_('unknown operating system: %s\n') % os.name)
     return path
 
 def inusercache(ui, hash):
     path = usercachepath(ui, hash)
     return path and os.path.exists(path)
 
 def findfile(repo, hash):
     if instore(repo, hash):
         repo.ui.note(_('found %s in store\n') % hash)
         return storepath(repo, hash)
     elif inusercache(repo.ui, hash):
         repo.ui.note(_('found %s in system cache\n') % hash)
         path = storepath(repo, hash)
         util.makedirs(os.path.dirname(path))
         link(usercachepath(repo.ui, hash), path)
         return path
     return None
 
 class largefilesdirstate(dirstate.dirstate):
     def __getitem__(self, key):
         return super(largefilesdirstate, self).__getitem__(unixpath(key))
     def normal(self, f):
         return super(largefilesdirstate, self).normal(unixpath(f))
     def remove(self, f):
         return super(largefilesdirstate, self).remove(unixpath(f))
     def add(self, f):
         return super(largefilesdirstate, self).add(unixpath(f))
     def drop(self, f):
         return super(largefilesdirstate, self).drop(unixpath(f))
     def forget(self, f):
         return super(largefilesdirstate, self).forget(unixpath(f))
     def normallookup(self, f):
         return super(largefilesdirstate, self).normallookup(unixpath(f))
     def _ignore(self):
         return False
 
 def openlfdirstate(ui, repo, create=True):
     '''
     Return a dirstate object that tracks largefiles: i.e. its root is
     the repo root, but it is saved in .hg/largefiles/dirstate.
     '''
     lfstoredir = repo.join(longname)
     opener = scmutil.opener(lfstoredir)
     lfdirstate = largefilesdirstate(opener, ui, repo.root,
                                      repo.dirstate._validate)
 
     # If the largefiles dirstate does not exist, populate and create
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
     if create and not os.path.exists(os.path.join(lfstoredir, 'dirstate')):
         util.makedirs(lfstoredir)
         matcher = getstandinmatcher(repo)
         for standin in repo.dirstate.walk(matcher, [], False, False):
             lfile = splitstandin(standin)
             hash = readstandin(repo, lfile)
             lfdirstate.normallookup(lfile)
             try:
                 if hash == hashfile(repo.wjoin(lfile)):
                     lfdirstate.normal(lfile)
             except OSError, err:
                 if err.errno != errno.ENOENT:
                     raise
     return lfdirstate
 
 def lfdirstatestatus(lfdirstate, repo, rev):
     match = match_.always(repo.root, repo.getcwd())
     s = lfdirstate.status(match, [], False, False, False)
     unsure, modified, added, removed, missing, unknown, ignored, clean = s
     for lfile in unsure:
         try:
             fctx = repo[rev][standin(lfile)]
         except LookupError:
             fctx = None
         if not fctx or fctx.data().strip() != hashfile(repo.wjoin(lfile)):
             modified.append(lfile)
         else:
             clean.append(lfile)
             lfdirstate.normal(lfile)
     return (modified, added, removed, missing, unknown, ignored, clean)
 
 def listlfiles(repo, rev=None, matcher=None):
     '''return a list of largefiles in the working copy or the
     specified changeset'''
 
     if matcher is None:
         matcher = getstandinmatcher(repo)
 
     # ignore unknown files in working directory
     return [splitstandin(f)
             for f in repo[rev].walk(matcher)
             if rev is not None or repo.dirstate[f] != '?']
 
 def instore(repo, hash):
     return os.path.exists(storepath(repo, hash))
 
 def storepath(repo, hash):
     return repo.join(os.path.join(longname, hash))
 
 def copyfromcache(repo, hash, filename):
     '''Copy the specified largefile from the repo or system cache to
     filename in the repository. Return true on success or false if the
     file was not found in either cache (which should not happened:
     this is meant to be called only after ensuring that the needed
     largefile exists in the cache).'''
     path = findfile(repo, hash)
     if path is None:
         return False
     util.makedirs(os.path.dirname(repo.wjoin(filename)))
     # The write may fail before the file is fully written, but we
     # don't use atomic writes in the working copy.
     shutil.copy(path, repo.wjoin(filename))
     return True
 
 def copytostore(repo, rev, file, uploaded=False):
     hash = readstandin(repo, file, rev)
     if instore(repo, hash):
         return
     copytostoreabsolute(repo, repo.wjoin(file), hash)
 
 def copyalltostore(repo, node):
     '''Copy all largefiles in a given revision to the store'''
 
     ctx = repo[node]
     for filename in ctx.files():
         if isstandin(filename) and filename in ctx.manifest():
             realfile = splitstandin(filename)
             copytostore(repo, ctx.node(), realfile)
 
 
 def copytostoreabsolute(repo, file, hash):
     util.makedirs(os.path.dirname(storepath(repo, hash)))
     if inusercache(repo.ui, hash):
         link(usercachepath(repo.ui, hash), storepath(repo, hash))
     elif not getattr(repo, "_isconverting", False):
         dst = util.atomictempfile(storepath(repo, hash),
                                   createmode=repo.store.createmode)
         for chunk in util.filechunkiter(open(file, 'rb')):
             dst.write(chunk)
         dst.close()
         linktousercache(repo, hash)
 
 def linktousercache(repo, hash):
     path = usercachepath(repo.ui, hash)
     if path:
         util.makedirs(os.path.dirname(path))
         link(storepath(repo, hash), path)
 
 def getstandinmatcher(repo, pats=[], opts={}):
     '''Return a match object that applies pats to the standin directory'''
     standindir = repo.wjoin(shortname)
     if pats:
         pats = [os.path.join(standindir, pat) for pat in pats]
-    elif os.path.isdir(standindir):
+    else:
         # no patterns: relative to repo root
         pats = [standindir]
-    else:
-        # no patterns and no standin dir: return matcher that matches nothing
-        return match_.match(repo.root, None, [], exact=True)
-
     # no warnings about missing files or directories
     match = scmutil.match(repo[None], pats, opts)
     match.bad = lambda f, msg: None
     return match
 
 def composestandinmatcher(repo, rmatcher):
     '''Return a matcher that accepts standins corresponding to the
     files accepted by rmatcher. Pass the list of files in the matcher
     as the paths specified by the user.'''
     smatcher = getstandinmatcher(repo, rmatcher.files())
     isstandin = smatcher.matchfn
     def composedmatchfn(f):
         return isstandin(f) and rmatcher.matchfn(splitstandin(f))
     smatcher.matchfn = composedmatchfn
 
     return smatcher
 
 def standin(filename):
     '''Return the repo-relative path to the standin for the specified big
     file.'''
     # Notes:
     # 1) Some callers want an absolute path, but for instance addlargefiles
     #    needs it repo-relative so it can be passed to repo[None].add().  So
     #    leave it up to the caller to use repo.wjoin() to get an absolute path.
     # 2) Join with '/' because that's what dirstate always uses, even on
     #    Windows. Change existing separator to '/' first in case we are
     #    passed filenames from an external source (like the command line).
     return shortnameslash + util.pconvert(filename)
 
 def isstandin(filename):
     '''Return true if filename is a big file standin. filename must be
     in Mercurial's internal form (slash-separated).'''
     return filename.startswith(shortnameslash)
 
 def splitstandin(filename):
     # Split on / because that's what dirstate always uses, even on Windows.
     # Change local separator to / first just in case we are passed filenames
     # from an external source (like the command line).
     bits = util.pconvert(filename).split('/', 1)
     if len(bits) == 2 and bits[0] == shortname:
         return bits[1]
     else:
         return None
 
 def updatestandin(repo, standin):
     file = repo.wjoin(splitstandin(standin))
     if os.path.exists(file):
         hash = hashfile(file)
         executable = getexecutable(file)
         writestandin(repo, standin, hash, executable)
 
 def readstandin(repo, filename, node=None):
     '''read hex hash from standin for filename at given node, or working
     directory if no node is given'''
     return repo[node][standin(filename)].data().strip()
 
 def writestandin(repo, standin, hash, executable):
     '''write hash to <repo.root>/<standin>'''
     writehash(hash, repo.wjoin(standin), executable)
 
 def copyandhash(instream, outfile):
     '''Read bytes from instream (iterable) and write them to outfile,
     computing the SHA-1 hash of the data along the way.  Close outfile
     when done and return the binary hash.'''
     hasher = util.sha1('')
     for data in instream:
         hasher.update(data)
         outfile.write(data)
 
     # Blecch: closing a file that somebody else opened is rude and
     # wrong. But it's so darn convenient and practical! After all,
     # outfile was opened just to copy and hash.
     outfile.close()
 
     return hasher.digest()
 
 def hashrepofile(repo, file):
     return hashfile(repo.wjoin(file))
 
 def hashfile(file):
     if not os.path.exists(file):
         return ''
     hasher = util.sha1('')
     fd = open(file, 'rb')
     for data in blockstream(fd):
         hasher.update(data)
     fd.close()
     return hasher.hexdigest()
 
 class limitreader(object):
     def __init__(self, f, limit):
         self.f = f
         self.limit = limit
 
     def read(self, length):
         if self.limit == 0:
             return ''
         length = length > self.limit and self.limit or length
         self.limit -= length
         return self.f.read(length)
 
     def close(self):
         pass
 
 def blockstream(infile, blocksize=128 * 1024):
     """Generator that yields blocks of data from infile and closes infile."""
     while True:
         data = infile.read(blocksize)
         if not data:
             break
         yield data
     # same blecch as copyandhash() above
     infile.close()
 
 def writehash(hash, filename, executable):
     util.makedirs(os.path.dirname(filename))
     util.writefile(filename, hash + '\n')
     os.chmod(filename, getmode(executable))
 
 def getexecutable(filename):
     mode = os.stat(filename).st_mode
     return ((mode & stat.S_IXUSR) and
             (mode & stat.S_IXGRP) and
             (mode & stat.S_IXOTH))
 
 def getmode(executable):
     if executable:
         return 0755
     else:
         return 0644
 
 def urljoin(first, second, *arg):
     def join(left, right):
         if not left.endswith('/'):
             left += '/'
         if right.startswith('/'):
             right = right[1:]
         return left + right
 
     url = join(first, second)
     for a in arg:
         url = join(url, a)
     return url
 
 def hexsha1(data):
     """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
     object data"""
     h = util.sha1()
     for chunk in util.filechunkiter(data):
         h.update(chunk)
     return h.hexdigest()
 
 def httpsendfile(ui, filename):
     return httpconnection.httpsendfile(ui, filename, 'rb')
 
 def unixpath(path):
     '''Return a version of path normalized for use with the lfdirstate.'''
     return util.pconvert(os.path.normpath(path))
 
 def islfilesrepo(repo):
     if ('largefiles' in repo.requirements and
             util.any(shortnameslash in f[0] for f in repo.store.datafiles())):
         return True
 
     return util.any(openlfdirstate(repo.ui, repo, False))
 
 class storeprotonotcapable(Exception):
     def __init__(self, storetypes):
         self.storetypes = storetypes
 
 def getcurrentheads(repo):
     branches = repo.branchmap()
     heads = []
     for branch in branches:
         newheads = repo.branchheads(branch)
         heads = heads + newheads
     return heads
 
 def getstandinsstate(repo):
     standins = []
     matcher = getstandinmatcher(repo)
     for standin in repo.dirstate.walk(matcher, [], False, False):
         lfile = splitstandin(standin)
         try:
             hash = readstandin(repo, lfile)
         except IOError:
             hash = None
         standins.append((lfile, hash))
     return standins
 
 def getlfilestoupdate(oldstandins, newstandins):
     changedstandins = set(oldstandins).symmetric_difference(set(newstandins))
     filelist = []
     for f in changedstandins:
         if f[0] not in filelist:
             filelist.append(f[0])
     return filelist

tests/test-issue3084.t

@@ -1,110 +1,112 b''
 
   $ echo "[extensions]" >> $HGRCPATH
   $ echo "largefiles =" >> $HGRCPATH
 
 Create the repository outside $HOME since largefiles write to
 $HOME/.cache/largefiles.
 
   $ hg init test
   $ cd test
   $ echo "root" > root
   $ hg add root
   $ hg commit -m "Root commit"
 
   $ echo "large" > foo
   $ hg add --large foo
   $ hg commit -m "Add foo as a largefile"
 
   $ hg update -r 0
   getting changed largefiles
   0 largefiles updated, 1 removed
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
 
   $ echo "normal" > foo
   $ hg add foo
   $ hg commit -m "Add foo as normal file"
   created new head
 
 Normal file in the working copy, keeping the normal version:
 
   $ echo "n" | hg merge --config ui.interactive=Yes
   foo has been turned into a largefile
   use (l)argefile or keep as (n)ormal file? 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
+  getting changed largefiles
+  0 largefiles updated, 0 removed
 
   $ hg status
   $ cat foo
   normal
 
 Normal file in the working copy, keeping the largefile version:
 
   $ hg update -q -C
   $ echo "l" | hg merge --config ui.interactive=Yes
   foo has been turned into a largefile
   use (l)argefile or keep as (n)ormal file? 1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
   1 largefiles updated, 0 removed
 
   $ hg status
   M foo
 
   $ hg diff --nodates
   diff -r fa129ab6b5a7 .hglf/foo
   --- /dev/null
   +++ b/.hglf/foo
   @@ -0,0 +1,1 @@
   +7f7097b041ccf68cc5561e9600da4655d21c6d18
   diff -r fa129ab6b5a7 foo
   --- a/foo
   +++ /dev/null
   @@ -1,1 +0,0 @@
   -normal
 
   $ cat foo
   large
 
 Largefile in the working copy, keeping the normal version:
 
   $ hg update -q -C -r 1
   $ echo "n" | hg merge --config ui.interactive=Yes
   foo has been turned into a normal file
   keep as (l)argefile or use (n)ormal file? 1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
   0 largefiles updated, 0 removed
 
   $ hg status
   M foo
 
   $ hg diff --nodates
   diff -r ff521236428a .hglf/foo
   --- a/.hglf/foo
   +++ /dev/null
   @@ -1,1 +0,0 @@
   -7f7097b041ccf68cc5561e9600da4655d21c6d18
   diff -r ff521236428a foo
   --- /dev/null
   +++ b/foo
   @@ -0,0 +1,1 @@
   +normal
 
   $ cat foo
   normal
 
 Largefile in the working copy, keeping the largefile version:
 
   $ hg update -q -C -r 1
   $ echo "l" | hg merge --config ui.interactive=Yes
   foo has been turned into a normal file
   keep as (l)argefile or use (n)ormal file? 0 files updated, 0 files merged, 1 files removed, 0 files unresolved
   (branch merge, don't forget to commit)
   getting changed largefiles
   1 largefiles updated, 0 removed
 
   $ hg status
 
   $ cat foo
   large
 
   $ cd ..

tests/test-largefiles-cache.t

@@ -1,127 +1,141 b''
 Create user cache directory
 
   $ USERCACHE=`pwd`/cache; export USERCACHE
   $ cat <<EOF >> ${HGRCPATH}
   > [extensions]
   > hgext.largefiles=
   > [largefiles]
   > usercache=${USERCACHE}
   > EOF
   $ mkdir -p ${USERCACHE}
 
 Create source repo, and commit adding largefile.
 
   $ hg init src
   $ cd src
   $ echo large > large
   $ hg add --large large
   $ hg commit -m 'add largefile'
   $ hg rm large
   $ hg commit -m 'branchhead without largefile'
   $ hg up -qr 0
   $ cd ..
 
 Discard all cached largefiles in USERCACHE
 
   $ rm -rf ${USERCACHE}
 
 Create mirror repo, and pull from source without largefile:
 "pull" is used instead of "clone" for suppression of (1) updating to
 tip (= cahcing largefile from source repo), and (2) recording source
 repo as "default" path in .hg/hgrc.
 
   $ hg init mirror
   $ cd mirror
   $ hg pull ../src
   pulling from ../src
   requesting all changes
   adding changesets
   adding manifests
   adding file changes
   added 2 changesets with 1 changes to 1 files
   (run 'hg update' to get a working copy)
   caching new largefiles
   0 largefiles cached
 
 Update working directory to "tip", which requires largefile("large"),
 but there is no cache file for it.  So, hg must treat it as
 "missing"(!) file.
 
   $ hg update -r0
   getting changed largefiles
   error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status
   ! large
 
 Update working directory to null: this cleanup .hg/largefiles/dirstate
 
   $ hg update null
   getting changed largefiles
   0 largefiles updated, 0 removed
   0 files updated, 0 files merged, 1 files removed, 0 files unresolved
 
 Update working directory to tip, again.
 
   $ hg update -r0
   getting changed largefiles
   error getting id 7f7097b041ccf68cc5561e9600da4655d21c6d18 from url file:$TESTTMP/mirror for file large: can't get file locally (glob)
   0 largefiles updated, 0 removed
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ hg status
   ! large
   $ cd ..
 
+Verify that largefiles from pulled branchheads are fetched, also to an empty repo
+
+  $ hg init mirror2
+  $ hg -R mirror2 pull src -r0
+  pulling from src
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files
+  (run 'hg update' to get a working copy)
+  caching new largefiles
+  abort: *: '$TESTTMP/mirror2/.hg/largefiles/.7f7097b041ccf68cc5561e9600da4655d21c6d18.*' (glob)
+  [255]
+
 #if unix-permissions
 
 Portable way to print file permissions:
 
   $ cat > ls-l.py <<EOF
   > #!/usr/bin/env python
   > import sys, os
   > path = sys.argv[1]
   > print '%03o' % (os.lstat(path).st_mode & 0777)
   > EOF
   $ chmod +x ls-l.py
 
 Test that files in .hg/largefiles inherit mode from .hg/store, not
 from file in working copy:
 
   $ cd src
   $ chmod 750 .hg/store
   $ chmod 660 large
   $ echo change >> large
   $ hg commit -m change
   created new head
   $ ../ls-l.py .hg/largefiles/e151b474069de4ca6898f67ce2f2a7263adf8fea
   640
 
 Test permission of with files in .hg/largefiles created by update:
 
   $ cd ../mirror
   $ rm -r "$USERCACHE" .hg/largefiles # avoid links
   $ chmod 750 .hg/store
   $ hg pull ../src --update -q
   $ ../ls-l.py .hg/largefiles/e151b474069de4ca6898f67ce2f2a7263adf8fea
   640
 
 Test permission of files created by push:
 
   $ hg serve -R ../src -d -p $HGPORT --pid-file hg.pid \
   >          --config "web.allow_push=*" --config web.push_ssl=no
   $ cat hg.pid >> $DAEMON_PIDS
 
   $ echo change >> large
   $ hg commit -m change
 
   $ rm -r "$USERCACHE"
 
   $ hg push -q http://localhost:$HGPORT/
 
   $ ../ls-l.py ../src/.hg/largefiles/b734e14a0971e370408ab9bce8d56d8485e368a9
   640
 
   $ cd ..
 
 #endif